From 61ac4f57394e022382459235baa598ac89ce00d1 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 23:03:56 +0000 Subject: [PATCH 01/30] test(e2e): add scripted-provider cost calculation suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 3 +- tests/e2e/conftest.py | 9 +- tests/e2e/cost_calculation/conftest.py | 139 ++++ tests/e2e/cost_calculation/cost_matrix.py | 458 +++++++++++++ tests/e2e/cost_calculation/scripted_client.py | 70 ++ .../e2e/cost_calculation/scripted_provider.py | 631 ++++++++++++++++++ .../test_token_pricing_e2e.py | 115 ++++ .../cost_calculation/test_wire_formats_e2e.py | 186 ++++++ tests/e2e/cost_map.json | 352 ++++++++++ .../coverage_registry/quota_management.yaml | 2 + tests/e2e/e2e_config.py | 16 + tests/e2e/pytest.ini | 1 + 12 files changed, 1980 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/cost_calculation/conftest.py create mode 100644 tests/e2e/cost_calculation/cost_matrix.py create mode 100644 tests/e2e/cost_calculation/scripted_client.py create mode 100644 tests/e2e/cost_calculation/scripted_provider.py create mode 100644 tests/e2e/cost_calculation/test_token_pricing_e2e.py create mode 100644 tests/e2e/cost_calculation/test_wire_formats_e2e.py create mode 100644 tests/e2e/cost_map.json diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 0541ce25d4b..b6c3840f626 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,6 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` @@ -221,7 +222,7 @@ other... ``` ## Hard Rules -- no unit tests of any kind under `tests/e2e`. a product feature is proven end to end against a live proxy, never with a unit test, and the harness itself is not unit-tested here either. no monkeypatching or mock tests. if a contributor asks you to write an end to end test, do NOT stage a unit test with it; if you find a product gap, call it out in the PR description +- no unit tests of any kind under `tests/e2e`. a product feature is proven end to end against a live proxy, never with a unit test, and the harness itself is not unit-tested here either. no monkeypatching or mock tests; the one carve-out is a scripted upstream served through a real HTTP sidecar (the cost_calculation suite's scripted provider), allowed because provider-response-shape coverage needs a controlled usage payload and every hop from the proxy's upstream call to the spend row still executes for real. if a contributor asks you to write an end to end test, do NOT stage a unit test with it; if you find a product gap, call it out in the PR description - use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want. diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b1a75d5f862..7ab41b8ff68 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -22,9 +22,9 @@ from typing import Final import pytest import requests - from e2e_config import ( CONTROL_PLANE_BASE_URL, + COST_MAP_OPT_IN_ENV, FIXTURE_DIR, FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, @@ -53,6 +53,7 @@ OPT_IN_MARKERS: Final = MappingProxyType( "managed_files": MANAGED_FILES_OPT_IN_ENV, "prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV, "redis_chaos": REDIS_CHAOS_OPT_IN_ENV, + "cost_map_stack": COST_MAP_OPT_IN_ENV, } ) @@ -120,6 +121,12 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) + config.addinivalue_line( + "markers", + "cost_map_stack: needs a proxy whose whole cost map is tests/e2e/cost_map.json " + "(LITELLM_MODEL_COST_MAP_URL) plus a scripted-provider sidecar; deselected unless " + "E2E_COST_MAP_STACK is set", + ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py new file mode 100644 index 00000000000..1bba3d50e1d --- /dev/null +++ b/tests/e2e/cost_calculation/conftest.py @@ -0,0 +1,139 @@ +"""Cost-calculation suite fixtures. + +Runs against a dedicated proxy whose whole model cost map is the test-owned +``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL), so every deployment +bills at rates the test asserts literal arithmetic on. Provider calls are +answered by the scripted-provider sidecar (``scripted_provider.py``), registered +per scenario over its control API. + +Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). +""" + +from __future__ import annotations + +import importlib.util +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType +from typing import Final, Protocol, cast + +import pytest + +from cost_matrix import Case, FrontierModel +from e2e_config import COST_MAP_PROXY_URL +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, ModelInfoBody, ModelNewBody +from proxy_client import ProxyClient, build_proxy_client +from scripted_client import ScenarioHandle, delete_scenario, register_scenario +from scripted_provider import Scenario + + +def _load_cost_rows() -> ModuleType: + """Load quota_management/spend_tracking/cost_rows.py by path (the e2e tree + has no package layout), the same trick the mcp suite uses for + logging/datadog_reader.py.""" + path = ( + Path(__file__).resolve().parent.parent + / "quota_management" + / "spend_tracking" + / "cost_rows.py" + ) + name = "e2e_spend_tracking_cost_rows" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +class SpendCostBreakdown(Protocol): + input_cost: float | None + output_cost: float | None + cache_read_cost: float | None + cache_creation_cost: float | None + reasoning_cost: float | None + tool_usage_cost: float | None + total_cost: float | None + service_tier: str | None + + def model_dump(self) -> dict[str, object]: ... + + +class SpendRowMetadata(Protocol): + cost_breakdown: SpendCostBreakdown | None + + +class SpendCostRow(Protocol): + """The slice of spend_tracking.cost_rows.CostRow this suite reads.""" + + spend: float | None + prompt_tokens: int | None + completion_tokens: int | None + metadata: SpendRowMetadata | None + + @property + def breakdown(self) -> SpendCostBreakdown: ... + + +class CostRowsModule(Protocol): + """cost_rows.py loaded by path has no importable name for basedpyright, so + its surface is declared here and reached through a single cast.""" + + approx_equal: Callable[[float, float], bool] + assert_total_is_sum_of_components: Callable[[SpendCostRow], None] + poll_cost_row_where: Callable[ + [ProxyClient, str, Callable[[SpendCostRow], bool]], SpendCostRow | None + ] + + +cost_rows: Final[CostRowsModule] = cast(CostRowsModule, _load_cost_rows()) + + +@dataclass(frozen=True, slots=True) +class CostCalcClient: + """The suite's client: a ProxyClient pointed at the cost-map proxy pod.""" + + proxy: ProxyClient + + +@pytest.fixture(scope="session") +def client() -> CostCalcClient: + proxy = build_proxy_client( + base_url=COST_MAP_PROXY_URL, + control_plane_base_url=COST_MAP_PROXY_URL, + replica_urls=(COST_MAP_PROXY_URL,), + ) + return CostCalcClient(proxy=proxy) + + +def register_scenario_deployment( + client: CostCalcClient, + resources: ResourceManager, + model: FrontierModel, + case: Case, + marker: str, +) -> tuple[str, ScenarioHandle]: + """Register the case's scenario on the sidecar plus a deployment pointed at + it; both are torn down by ``resources``. Returns the callable model_name.""" + scenario: Scenario = case.scenario( + scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" + ) + handle = register_scenario(scenario) + resources.defer(lambda: delete_scenario(handle)) + model_name = f"{model.model_name}-{marker}" + model_id = client.proxy.register_model( + ModelNewBody( + model_name=model_name, + litellm_params=LiteLLMParamsBody( + model=model.litellm_model, + api_key="sk-scripted-provider", + api_base=handle.api_base(), + ), + model_info=ModelInfoBody(), + ) + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model_name, handle diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py new file mode 100644 index 00000000000..bc466d7d823 --- /dev/null +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -0,0 +1,458 @@ +"""The cost-calculation matrix: frontier model set, the pricing-component cases +each model runs, and the expected-cost arithmetic. + +Rates come from ``tests/e2e/cost_map.json``, which the proxy under test loads as +its ENTIRE model cost map (LITELLM_MODEL_COST_MAP_URL), so an entry's rates are +exactly what the proxy bills and nothing in the suite depends on the bundled +map. Each model's rates are a distinct multiple of a shared base set, so a +component billed at the wrong model's rate (or the wrong case's rate) can never +coincidentally match. + +Case applicability is pricing-field-gated AND wire-gated: a case runs for a +model only when the entry carries the rate the case exercises and the wire can +report the token kind that rate prices. When the wire cannot report a kind +(e.g. Anthropic has no reasoning-token field, Responses reports no cache +creation), the case is absent from the matrix rather than silently zero. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from scripted_provider import Scenario, ScriptedOutput, ScriptedUsage, Wire + +COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" + + +class SearchContextCostPerQuery(BaseModel): + model_config = ConfigDict(frozen=True) + + search_context_size_low: float | None = None + search_context_size_medium: float | None = None + search_context_size_high: float | None = None + + +class CostMapEntry(BaseModel): + """The pricing fields of a cost-map entry the matrix reads. Shaped like a + ``model_prices_and_context_window.json`` entry; unmodelled keys are ignored.""" + + model_config = ConfigDict(frozen=True, extra="ignore") + + litellm_provider: str + mode: str + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + cache_creation_input_token_cost_above_1hr: float | None = None + output_cost_per_reasoning_token: float | None = None + input_cost_per_audio_token: float | None = None + output_cost_per_audio_token: float | None = None + input_cost_per_token_above_200k_tokens: float | None = None + output_cost_per_token_above_200k_tokens: float | None = None + input_cost_per_token_flex: float | None = None + output_cost_per_token_flex: float | None = None + input_cost_per_token_priority: float | None = None + output_cost_per_token_priority: float | None = None + search_context_cost_per_query: SearchContextCostPerQuery | None = None + web_search_billing_unit: str | None = None + + +_COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) +_COST_MAP: Final[dict[str, CostMapEntry]] = _COST_MAP_ADAPTER.validate_python( + json.loads(COST_MAP_PATH.read_text()) +) + +TIER_THRESHOLD_TOKENS: Final = 200_000 + + +@dataclass(frozen=True, slots=True) +class FrontierModel: + """One deployment under test: the model_name the suite registers, the + provider-prefixed litellm model string, the wire the scripted upstream + speaks, its cost-map key, and the sibling map model the response_model + override case reports.""" + + model_name: str + litellm_model: str + wire: Wire + map_key: str + override_model: str + + @property + def rates(self) -> CostMapEntry: + return _COST_MAP[self.map_key] + + @property + def override_rates(self) -> CostMapEntry: + return _COST_MAP[self.override_map_key] + + @property + def override_map_key(self) -> str: + return _OVERRIDE_MAP_KEYS[self.override_model] + + @property + def provider(self) -> str: + return self.rates.litellm_provider + + @property + def api_key(self) -> str: + # The scripted upstream ignores auth; a fixed bogus key proves the suite + # spends zero real provider calls. + return "sk-scripted-provider" + + +# Response-model override targets: emit a sibling's bare provider-facing name so +# the biller's provider-prefixed lookup lands on that sibling's map key. +_OVERRIDE_MODELS: Final[dict[str, str]] = { + "gpt-5.6": "gpt-5.4-mini", + "gpt-5.5-pro": "gpt-5.3-codex", + "gpt-5.3-codex": "gpt-5.5-pro", + "gpt-5.4-mini": "gpt-5.6", + "claude-opus-5": "claude-sonnet-5", + "claude-sonnet-5": "claude-opus-5", + "claude-haiku-4-5": "claude-sonnet-5", + "gemini/gemini-3.8-flash": "gemini-3.1-pro-preview", + "gemini/gemini-3.1-pro-preview": "gemini-3.8-flash", + "together_ai/moonshotai/Kimi-K3": "zai-org/GLM-5.3", + "together_ai/zai-org/GLM-5.3": "moonshotai/Kimi-K3", + "fireworks_ai/kimi-k3": "qwen3p8-max", + "fireworks_ai/qwen3p8-max": "kimi-k3", + "fireworks_ai/deepseek-v4p1-flash": "kimi-k3", +} + +_OVERRIDE_MAP_KEYS: Final[dict[str, str]] = { + "gpt-5.4-mini": "gpt-5.4-mini", + "gpt-5.6": "gpt-5.6", + "gpt-5.3-codex": "gpt-5.3-codex", + "gpt-5.5-pro": "gpt-5.5-pro", + "claude-sonnet-5": "claude-sonnet-5", + "claude-opus-5": "claude-opus-5", + "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview", + "gemini-3.8-flash": "gemini/gemini-3.8-flash", + "zai-org/GLM-5.3": "together_ai/zai-org/GLM-5.3", + "moonshotai/Kimi-K3": "together_ai/moonshotai/Kimi-K3", + "qwen3p8-max": "fireworks_ai/qwen3p8-max", + "kimi-k3": "fireworks_ai/kimi-k3", +} + + +_FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( + ("gpt-5.6", "openai/gpt-5.6", "openai_chat"), + ("gpt-5.5-pro", "openai/gpt-5.5-pro", "openai_responses"), + ("gpt-5.3-codex", "openai/gpt-5.3-codex", "openai_responses"), + ("gpt-5.4-mini", "openai/gpt-5.4-mini", "openai_chat"), + ("claude-opus-5", "anthropic/claude-opus-5", "anthropic_messages"), + ("claude-sonnet-5", "anthropic/claude-sonnet-5", "anthropic_messages"), + ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "anthropic_messages"), + ("gemini/gemini-3.8-flash", "gemini/gemini-3.8-flash", "gemini_generate"), + ("gemini/gemini-3.1-pro-preview", "gemini/gemini-3.1-pro-preview", "gemini_generate"), + ("together_ai/moonshotai/Kimi-K3", "together_ai/moonshotai/Kimi-K3", "together_chat"), + ("together_ai/zai-org/GLM-5.3", "together_ai/zai-org/GLM-5.3", "together_chat"), + ("fireworks_ai/kimi-k3", "fireworks_ai/kimi-k3", "fireworks_chat"), + ("fireworks_ai/qwen3p8-max", "fireworks_ai/qwen3p8-max", "fireworks_chat"), + ("fireworks_ai/deepseek-v4p1-flash", "fireworks_ai/deepseek-v4p1-flash", "fireworks_chat"), +) + + +def _frontier() -> tuple[FrontierModel, ...]: + return tuple( + FrontierModel( + model_name=f"cc-{map_key.replace('/', '-').lower()}", + litellm_model=litellm_model, + wire=wire, + map_key=map_key, + override_model=_OVERRIDE_MODELS[map_key], + ) + for map_key, litellm_model, wire in _FRONTIER_SPECS + ) + + +FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() + +# Token kinds each wire can report, gating which pricing cases apply. +_WIRE_CAPS: Final[dict[str, frozenset[str]]] = { + "openai_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", + } + ), + "openai_responses": frozenset({"cache_read", "reasoning", "web_search", "response_model", "absent_usage"}), + # Product gap: litellm hard-indexes message_delta["usage"] in + # anthropic/chat/handler.py, so a usage-absent anthropic stream raises + # KeyError; the real wire always carries it, so the case cannot be + # represented. + "anthropic_messages": frozenset({"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model"}), + # Product gap: the gemini transform sets ModelResponse.model from the + # request and drops the provider's modelVersion, so a response-model + # override can never be priced on this wire. + "gemini_generate": frozenset({"cache_read", "reasoning", "audio", "web_search", "absent_usage"}), + "together_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", + } + ), + "fireworks_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", + } + ), +} + +CaseName = Literal[ + "basic", + "cache_read", + "cache_write_5m", + "cache_write_1h", + "reasoning", + "audio", + "tiered", + "service_tier_flex", + "service_tier_priority", + "web_search", + "stream", + "stream_no_usage", + "response_model_override", +] + + +@dataclass(frozen=True, slots=True) +class Case: + name: CaseName + usage: ScriptedUsage + stream: bool = False + stream_usage: Literal["final_chunk", "absent"] = "final_chunk" + service_tier: Literal["flex", "priority"] | None = None + # For web_search the wire's reported call count is not always what gets + # billed: chat-completions surfaces only expose url_citation annotations, so + # the biller floors to one call; responses/messages/gemini report a real + # count. + billed_web_search_calls: int = 0 + response_model_override: bool = False + exact_spend: bool = True + # stream_usage=absent on a wire with no proxy-side token recount means the + # bill is exactly zero; asserted as such rather than skipped. + expect_zero_bill: bool = False + + def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: + return Scenario( + scenario_id=scenario_id, + wire=model.wire, + usage=self.usage, + output=ScriptedOutput( + text=text, + response_model=model.override_model if self.response_model_override else None, + ), + stream_usage=self.stream_usage, + service_tier=self.service_tier, + ) + + +_BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) + + +def _web_search_case(model: FrontierModel) -> Case: + counts_exactly = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") + return Case( + name="web_search", + usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), + billed_web_search_calls=3 if counts_exactly else 1, + ) + + +def cases_for(model: FrontierModel) -> tuple[Case, ...]: + rates = model.rates + caps = _WIRE_CAPS[model.wire] + cases: list[Case] = [Case(name="basic", usage=_BASIC_USAGE)] + if rates.cache_read_input_token_cost is not None and "cache_read" in caps: + cases.append( + Case(name="cache_read", usage=ScriptedUsage(fresh_input_tokens=100, cache_read_tokens=50, output_tokens=30)) + ) + if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps: + cases.append( + Case( + name="cache_write_5m", + usage=ScriptedUsage(fresh_input_tokens=90, cache_write_5m_tokens=60, output_tokens=30), + ) + ) + if ( + rates.cache_creation_input_token_cost_above_1hr is not None + and rates.cache_creation_input_token_cost is not None + and "cache_write_1h" in caps + ): + cases.append( + Case( + name="cache_write_1h", + usage=ScriptedUsage( + fresh_input_tokens=90, + cache_write_5m_tokens=20, + cache_write_1h_tokens=40, + output_tokens=30, + ), + ) + ) + if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps: + cases.append( + Case( + name="reasoning", + usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, reasoning_tokens=70), + ) + ) + if ( + rates.input_cost_per_audio_token is not None + and rates.output_cost_per_audio_token is not None + and "audio" in caps + ): + cases.append( + Case( + name="audio", + usage=ScriptedUsage( + fresh_input_tokens=100, audio_input_tokens=25, output_tokens=30, audio_output_tokens=15 + ), + ) + ) + if ( + rates.input_cost_per_token_above_200k_tokens is not None + and rates.output_cost_per_token_above_200k_tokens is not None + ): + cases.append( + Case( + name="tiered", + usage=ScriptedUsage( + fresh_input_tokens=TIER_THRESHOLD_TOKENS + 1, output_tokens=30 + ), + ) + ) + if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None: + cases.append( + Case(name="service_tier_flex", usage=_BASIC_USAGE, service_tier="flex") + ) + if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None: + cases.append( + Case(name="service_tier_priority", usage=_BASIC_USAGE, service_tier="priority") + ) + if rates.search_context_cost_per_query is not None and "web_search" in caps: + cases.append(_web_search_case(model)) + cases.append(Case(name="stream", usage=_BASIC_USAGE, stream=True)) + if "absent_usage" in caps: + cases.append( + Case( + name="stream_no_usage", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + exact_spend=False, + # The responses surface bills only provider-reported usage; + # with no usage in the stream the spend row is zero. Other + # wires recount tokens proxy-side and bill a nonzero amount. + expect_zero_bill=model.wire == "openai_responses", + ) + ) + if "response_model" in caps: + cases.append(Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True)) + return tuple(cases) + + +@dataclass(frozen=True, slots=True) +class ExpectedCost: + """The expected bill split the way the spend row's cost_breakdown reports + it: the gross input component (cache reads/writes folded in), the output + component, and the tool-usage component.""" + + input_cost: float + output_cost: float + tool_cost: float + + @property + def total(self) -> float: + return self.input_cost + self.output_cost + self.tool_cost + + +def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: + """Literal arithmetic on the test-map rates over the scripted token counts. + + Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; + output = text*out + reasoning*reasoning + audio_out*audio_out; plus the + billed web-search calls at the medium search-context rate. Above-threshold + swaps every input/output rate to its ``_above_200k_tokens`` variant when + total prompt tokens exceed the threshold; a service tier swaps input/output + to the tier's variants, falling back to the base rate when a variant is + unset -- mirroring _get_token_base_cost in litellm's cost calculator. + """ + rates = model.override_rates if case.response_model_override else model.rates + u = case.usage + prompt_tokens = ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + u.audio_input_tokens + ) + tiered = prompt_tokens > TIER_THRESHOLD_TOKENS + in_rate = rates.input_cost_per_token or 0.0 + out_rate = rates.output_cost_per_token or 0.0 + if case.service_tier == "flex": + in_rate = rates.input_cost_per_token_flex or in_rate + out_rate = rates.output_cost_per_token_flex or out_rate + if case.service_tier == "priority": + in_rate = rates.input_cost_per_token_priority or in_rate + out_rate = rates.output_cost_per_token_priority or out_rate + if tiered: + in_rate = rates.input_cost_per_token_above_200k_tokens or in_rate + out_rate = rates.output_cost_per_token_above_200k_tokens or out_rate + input_cost = ( + u.fresh_input_tokens * in_rate + + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) + + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) + + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) + ) + output_cost = ( + u.output_tokens * out_rate + + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) + + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) + ) + search = rates.search_context_cost_per_query + tool_cost = case.billed_web_search_calls * ( + search.search_context_size_medium if search and search.search_context_size_medium else 0.0 + ) + return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) + + +def expected_cost(model: FrontierModel, case: Case) -> float: + return expected_breakdown(model, case).total + + +def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: + """(prompt_tokens, completion_tokens) the spend row should carry, per the + wire's normalization: Anthropic folds cache read/write into prompt_tokens, + everyone else reports the totals the wire emitted.""" + u = case.usage + if model.wire == "anthropic_messages": + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, + u.output_tokens, + ) + if model.wire == "gemini_generate": + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) + if model.wire == "openai_responses": + return ( + u.fresh_input_tokens + u.cache_read_tokens, + u.output_tokens + u.reasoning_tokens, + ) + return ( + u.fresh_input_tokens + + u.cache_read_tokens + + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) diff --git a/tests/e2e/cost_calculation/scripted_client.py b/tests/e2e/cost_calculation/scripted_client.py new file mode 100644 index 00000000000..dceec02630a --- /dev/null +++ b/tests/e2e/cost_calculation/scripted_client.py @@ -0,0 +1,70 @@ +"""Client side of the scripted-provider sidecar: register scenarios over its +control API through the shared transport helpers and get back a handle whose +``api_base`` is what a /model/new deployment should register for the proxy to +reach the scripted wire.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +from e2e_config import SCRIPTED_PROVIDER_CONTROL_URL, SCRIPTED_PROVIDER_PROXY_BASE +from e2e_http import URL, NoBody, unwrap, post +from e2e_http import delete as http_delete +from scripted_provider import ( + Scenario, + ScenarioDeleted, + ScenarioRegistered, + Wire, +) + + +@dataclass(frozen=True, slots=True) +class ScenarioHandle: + scenario_id: str + wire: Wire + proxy_base: str + + def api_base(self) -> str: + return f"{self.proxy_base}/{self.scenario_id}/{self._mount()}" + + def _mount(self) -> str: + return { + "openai_chat": "openai", + "openai_responses": "openai", + "anthropic_messages": "anthropic", + "gemini_generate": "gemini", + "together_chat": "together", + "fireworks_chat": "fireworks", + }[self.wire] + + +def register_scenario(scenario: Scenario) -> ScenarioHandle: + """POST the scenario to the sidecar's control API and return its handle.""" + result = unwrap( + post( + URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios"), + headers=NoBody(), + json=scenario, + response_type=ScenarioRegistered, + ) + ) + return ScenarioHandle( + scenario_id=result.scenario_id, + wire=scenario.wire, + proxy_base=SCRIPTED_PROVIDER_PROXY_BASE, + ) + + +def delete_scenario(handle: ScenarioHandle) -> None: + unwrap( + http_delete( + URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios/{handle.scenario_id}"), + headers=NoBody(), + json=NoBody(), + response_type=ScenarioDeleted, + ) + ) + + +CONTROL_URL: Final = SCRIPTED_PROVIDER_CONTROL_URL diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py new file mode 100644 index 00000000000..93a6f49ec25 --- /dev/null +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -0,0 +1,631 @@ +"""Scripted provider sidecar for the cost-calculation e2e suite. + +A standalone process (``python -m cost_calculation.scripted_provider``) that +pretends to be an LLM provider for the proxy under test. The suite registers a +Scenario over a small control API; the provider wire routes then answer the +proxy's upstream calls with the scripted usage figures, in the exact wire shape +the real provider would emit (OpenAI chat completions, OpenAI Responses, +Anthropic Messages, Gemini generateContent, or the OpenAI-compatible Together / +Fireworks surfaces). Because the usage is scripted, expected spend is literal +arithmetic on the test cost map's rates, with no dependency on what a real +provider would report. + +Layout on one port: + +- ``GET /health`` liveness +- ``POST /_scenarios`` register a Scenario JSON, returns its id +- ``DELETE /_scenarios/`` remove it +- ``POST ///`` provider wire; mount is one of + ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks`` and the + remainder is whatever path the provider client appends (``chat/completions``, + ``responses``, ``v1/messages``, ``models/:generateContent`` ...) + +A request carrying ``"stream": true`` (or the ``:streamGenerateContent`` Gemini +verb) gets an SSE answer; ``stream_usage`` on the Scenario decides whether the +final stream chunk carries usage or the provider reports none. +""" + +from __future__ import annotations + +import json +import sys +import threading +import time +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, Literal +from urllib.parse import urlsplit + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +Wire = Literal[ + "openai_chat", + "openai_responses", + "anthropic_messages", + "gemini_generate", + "together_chat", + "fireworks_chat", +] + +_WIRE_MOUNTS: Final[dict[str, str]] = { + "openai_chat": "openai", + "openai_responses": "openai", + "anthropic_messages": "anthropic", + "gemini_generate": "gemini", + "together_chat": "together", + "fireworks_chat": "fireworks", +} + +StreamUsage = Literal["final_chunk", "absent"] +ServiceTier = Literal["flex", "priority"] + + +class ScriptedUsage(BaseModel): + """Physical token counts the scripted response reports. ``fresh_input_tokens`` + is the uncached, never-written, non-audio input count; ``output_tokens`` is + the non-reasoning, non-audio output count. Renderers add the cached, written, + audio, and reasoning counts into the wire's total fields the way the real + provider does (inside prompt_tokens for OpenAI/Gemini, as uncached-only + input_tokens for Anthropic).""" + + model_config = ConfigDict(frozen=True) + + fresh_input_tokens: int = 0 + output_tokens: int = 0 + cache_read_tokens: int = 0 + cache_write_5m_tokens: int = 0 + cache_write_1h_tokens: int = 0 + reasoning_tokens: int = 0 + audio_input_tokens: int = 0 + audio_output_tokens: int = 0 + web_search_calls: int = 0 + + +class ScriptedOutput(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + finish_reason: str = "stop" + # When set, emitted verbatim as the response's model field, letting a test + # prove the biller prices the provider-reported model. + response_model: str | None = None + # OpenAI-compatible providers can report a provider-computed cost; emitted as + # the top-level "cost" field on the together/fireworks wire. + provider_cost: float | None = None + + +class Scenario(BaseModel): + model_config = ConfigDict(frozen=True) + + scenario_id: str + wire: Wire + usage: ScriptedUsage + output: ScriptedOutput + stream_usage: StreamUsage = "final_chunk" + service_tier: ServiceTier | None = None + + @property + def mount(self) -> str: + return _WIRE_MOUNTS[self.wire] + + +class ScenarioRegistered(BaseModel): + scenario_id: str + + +class ScenarioDeleted(BaseModel): + deleted: bool + + +class HealthStatus(BaseModel): + status: str + + +@dataclass(frozen=True, slots=True) +class RenderedResponse: + status_code: int + content_type: str + body: bytes + + +def _json_bytes(payload: dict[str, object]) -> bytes: + return json.dumps(payload).encode("utf-8") + + +def _sse(events: tuple[tuple[str | None, dict[str, object] | str], ...]) -> bytes: + frames: list[str] = [] + for event_name, data in events: + head = f"event: {event_name}\n" if event_name is not None else "" + payload = data if isinstance(data, str) else json.dumps(data) + frames.append(f"{head}data: {payload}\n\n") + return "".join(frames).encode("utf-8") + + +# ---------- per-wire usage shapes ---------- + + +def _openai_usage(u: ScriptedUsage) -> dict[str, object]: + prompt_tokens = ( + u.fresh_input_tokens + + u.cache_read_tokens + + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + + u.audio_input_tokens + ) + completion_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + prompt_details: dict[str, object] = {} + if u.cache_read_tokens: + prompt_details["cached_tokens"] = u.cache_read_tokens + if u.cache_write_5m_tokens or u.cache_write_1h_tokens: + prompt_details["cache_write_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens + prompt_details["cache_creation_token_details"] = { + "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, + "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, + } + if u.audio_input_tokens: + prompt_details["audio_tokens"] = u.audio_input_tokens + completion_details: dict[str, object] = {} + if u.reasoning_tokens: + completion_details["reasoning_tokens"] = u.reasoning_tokens + if u.audio_output_tokens: + completion_details["audio_tokens"] = u.audio_output_tokens + usage: dict[str, object] = { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + } + if prompt_details: + usage["prompt_tokens_details"] = prompt_details + if completion_details: + usage["completion_tokens_details"] = completion_details + return usage + + +def _anthropic_usage(u: ScriptedUsage) -> dict[str, object]: + # Anthropic reports uncached-only input_tokens; cache reads and writes ride + # top-level fields, with the 5m/1h write split under cache_creation. + usage: dict[str, object] = { + "input_tokens": u.fresh_input_tokens, + "output_tokens": u.output_tokens, + } + if u.cache_read_tokens: + usage["cache_read_input_tokens"] = u.cache_read_tokens + if u.cache_write_5m_tokens or u.cache_write_1h_tokens: + usage["cache_creation_input_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens + usage["cache_creation"] = { + "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, + "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, + } + if u.web_search_calls: + usage["server_tool_use"] = {"web_search_requests": u.web_search_calls} + return usage + + +def _gemini_usage(u: ScriptedUsage) -> dict[str, object]: + # promptTokenCount carries the cached count inside it; TEXT modality is the + # cached-inclusive text count so litellm's implicit-caching subtraction lands + # on the fresh figure. candidatesTokenCount includes reasoning + audio. + prompt_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + candidates = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + usage: dict[str, object] = { + "promptTokenCount": prompt_tokens, + "candidatesTokenCount": candidates, + "totalTokenCount": prompt_tokens + candidates, + } + if u.cache_read_tokens: + usage["cachedContentTokenCount"] = u.cache_read_tokens + if u.reasoning_tokens: + usage["thoughtsTokenCount"] = u.reasoning_tokens + prompt_details = [{"modality": "TEXT", "tokenCount": u.fresh_input_tokens + u.cache_read_tokens}] + if u.audio_input_tokens: + prompt_details.append({"modality": "AUDIO", "tokenCount": u.audio_input_tokens}) + usage["promptTokensDetails"] = prompt_details + if u.audio_output_tokens: + usage["candidatesTokensDetails"] = [ + {"modality": "TEXT", "tokenCount": u.output_tokens + u.reasoning_tokens}, + {"modality": "AUDIO", "tokenCount": u.audio_output_tokens}, + ] + return usage + + +def _responses_usage(u: ScriptedUsage) -> dict[str, object]: + input_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + output_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + usage: dict[str, object] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + } + input_details: dict[str, object] = {} + if u.cache_read_tokens: + input_details["cached_tokens"] = u.cache_read_tokens + if input_details: + usage["input_tokens_details"] = input_details + if u.reasoning_tokens: + usage["output_tokens_details"] = {"reasoning_tokens": u.reasoning_tokens} + return usage + + +# ---------- per-wire responses ---------- + + +def _openai_message(scenario: Scenario) -> dict[str, object]: + message: dict[str, object] = {"role": "assistant", "content": scenario.output.text} + if scenario.usage.web_search_calls: + message["annotations"] = [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1, + }, + } + for _ in range(scenario.usage.web_search_calls) + ] + return message + + +def _openai_chat_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + body: dict[str, object] = { + "id": f"chatcmpl-{scenario.scenario_id}", + "object": "chat.completion", + "created": int(time.time()), + "model": scenario.output.response_model or requested_model, + "choices": [ + { + "index": 0, + "message": _openai_message(scenario), + "finish_reason": scenario.output.finish_reason, + } + ], + "usage": _openai_usage(scenario.usage), + } + if scenario.service_tier is not None: + body["service_tier"] = scenario.service_tier + if scenario.output.provider_cost is not None: + body["cost"] = scenario.output.provider_cost + return body + + +def _openai_chunk(scenario: Scenario, requested_model: str, **kw: object) -> dict[str, object]: + chunk: dict[str, object] = { + "id": f"chatcmpl-{scenario.scenario_id}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": scenario.output.response_model or requested_model, + } + chunk.update(kw) + return chunk + + +def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: + _EMPTY_DELTA: Final[dict[str, object]] = {} + delta: dict[str, object] = {"role": "assistant", "content": scenario.output.text} + if scenario.usage.web_search_calls: + delta["annotations"] = _openai_message(scenario)["annotations"] + events: list[tuple[str | None, dict[str, object] | str]] = [ + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=[{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], + ), + ), + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=[{"index": 0, "delta": delta, "finish_reason": None}], + ), + ), + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=[ + { + "index": 0, + "delta": _EMPTY_DELTA, + "finish_reason": scenario.output.finish_reason, + } + ], + ), + ), + ] + if scenario.stream_usage == "final_chunk": + events.append( + (None, _openai_chunk(scenario, requested_model, choices=(), usage=_openai_usage(scenario.usage))) + ) + events.append((None, "[DONE]")) + return _sse(tuple(events)) + + +def _anthropic_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + return { + "id": f"msg_{scenario.scenario_id}", + "type": "message", + "role": "assistant", + "model": scenario.output.response_model or requested_model, + "content": [{"type": "text", "text": scenario.output.text}], + "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, + "usage": _anthropic_usage(scenario.usage), + } + + +def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: + emit_usage = scenario.stream_usage == "final_chunk" + input_usage = {k: v for k, v in _anthropic_usage(scenario.usage).items() if k != "output_tokens"} + message_start: dict[str, object] = { + "type": "message_start", + "message": { + "id": f"msg_{scenario.scenario_id}", + "type": "message", + "role": "assistant", + "model": scenario.output.response_model or requested_model, + "content": [], + "stop_reason": None, + **({"usage": input_usage} if emit_usage else {}), + }, + } + message_delta: dict[str, object] = { + "type": "message_delta", + "delta": { + "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason + }, + **({"usage": {"output_tokens": scenario.usage.output_tokens}} if emit_usage else {}), + } + return _sse( + ( + ("message_start", message_start), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": scenario.output.text}, + }, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_delta", message_delta), + ("message_stop", {"type": "message_stop"}), + ) + ) + + +def _gemini_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + candidate: dict[str, object] = { + "content": {"parts": [{"text": scenario.output.text}], "role": "model"}, + "finishReason": "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), + "index": 0, + } + if scenario.usage.web_search_calls: + candidate["groundingMetadata"] = { + "webSearchQueries": [f"query {i}" for i in range(scenario.usage.web_search_calls)] + } + return { + "candidates": [candidate], + "usageMetadata": _gemini_usage(scenario.usage), + "modelVersion": scenario.output.response_model or requested_model, + } + + +def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: + first = _gemini_body(scenario, requested_model) + if scenario.stream_usage == "absent": + first = {k: v for k, v in first.items() if k != "usageMetadata"} + events: list[tuple[str | None, dict[str, object] | str]] = [(None, first)] + if scenario.stream_usage == "final_chunk": + events.append( + ( + None, + { + "candidates": [], + "usageMetadata": _gemini_usage(scenario.usage), + "modelVersion": scenario.output.response_model or requested_model, + }, + ) + ) + return _sse(tuple(events)) + + +def _responses_body(scenario: Scenario, requested_model: str) -> dict[str, object]: + output: list[dict[str, object]] = [ + {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} + for i in range(scenario.usage.web_search_calls) + ] + output.append( + { + "type": "message", + "id": f"msg_{scenario.scenario_id}", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": scenario.output.text, + "annotations": [], + } + ], + } + ) + return { + "id": f"resp_{scenario.scenario_id}", + "object": "response", + "created_at": int(time.time()), + "status": "completed", + "model": scenario.output.response_model or requested_model, + "output": output, + "usage": _responses_usage(scenario.usage), + } + + +def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: + completed = _responses_body(scenario, requested_model) + if scenario.stream_usage == "absent": + completed = {k: v for k, v in completed.items() if k != "usage"} + created = {**completed, "status": "in_progress", "usage": None} + return _sse( + ( + ("response.created", {"type": "response.created", "response": created}), + ( + "response.output_text.delta", + { + "type": "response.output_text.delta", + "item_id": f"msg_{scenario.scenario_id}", + "output_index": scenario.usage.web_search_calls, + "content_index": 0, + "delta": scenario.output.text, + }, + ), + ("response.completed", {"type": "response.completed", "response": completed}), + ) + ) + + +def _render(scenario: Scenario, *, stream: bool, requested_model: str) -> RenderedResponse: + if scenario.wire == "anthropic_messages": + if stream: + return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model))) + if scenario.wire == "gemini_generate": + if stream: + return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) + if scenario.wire == "openai_responses": + if stream: + return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) + # openai_chat, together_chat, fireworks_chat share the OpenAI chat shape. + if stream: + return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) + + +# ---------- registry + request routing ---------- + + +class _ScenarioStore: + def __init__(self) -> None: + self._lock: Final = threading.Lock() + self._scenarios: dict[str, Scenario] = {} # mutable-ok: server state, guarded by _lock + + def put(self, scenario: Scenario) -> None: + with self._lock: + self._scenarios[scenario.scenario_id] = scenario + + def drop(self, scenario_id: str) -> bool: + with self._lock: + return self._scenarios.pop(scenario_id, None) is not None + + def get(self, scenario_id: str) -> Scenario | None: + with self._lock: + return self._scenarios.get(scenario_id) + + +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + +def _request_body(body: bytes) -> dict[str, object]: + try: + return _REQUEST_BODY.validate_json(body) + except ValueError: + return {} + + +def _request_wants_stream(path_tail: str, body: bytes) -> bool: + if ":streamGenerateContent" in path_tail: + return True + if not body: + return False + return _request_body(body).get("stream") is True + + +def _request_model(body: bytes) -> str: + model = _request_body(body).get("model") + return model if isinstance(model, str) else "unknown" + + +def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: + path = urlsplit(raw_path).path + segments = [segment for segment in path.split("/") if segment] + if method == "GET" and segments == ["health"]: + return RenderedResponse(200, "application/json", _json_bytes({"status": "ok"})) + if segments and segments[0] == "_scenarios": + if method == "POST" and len(segments) == 1: + try: + scenario = Scenario.model_validate_json(body) + except ValidationError as exc: + return RenderedResponse(400, "application/json", _json_bytes({"error": str(exc)})) + store.put(scenario) + return RenderedResponse(200, "application/json", _json_bytes({"scenario_id": scenario.scenario_id})) + if method == "DELETE" and len(segments) == 2: + deleted = store.drop(segments[1]) + return RenderedResponse( + 200 if deleted else 404, "application/json", _json_bytes({"deleted": deleted}) + ) + return RenderedResponse(404, "application/json", _json_bytes({"error": "unknown control route"})) + if len(segments) < 2 or method != "POST": + return RenderedResponse(404, "application/json", _json_bytes({"error": f"no route for {method} {path}"})) + scenario_id, mount = segments[0], segments[1] + scenario = store.get(scenario_id) + if scenario is None: + return RenderedResponse(404, "application/json", _json_bytes({"error": f"unknown scenario {scenario_id}"})) + if scenario.mount != mount: + return RenderedResponse( + 400, + "application/json", + _json_bytes({"error": f"scenario {scenario_id} is wire {scenario.wire}, not mount {mount}"}), + ) + tail = "/".join(segments[2:]) + return _render(scenario, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) + + +class _ScriptedHandler(BaseHTTPRequestHandler): + store: Final[_ScenarioStore] = _ScenarioStore() + + def _dispatch(self, method: str) -> None: + length = int(self.headers.get("content-length") or 0) + body = self.rfile.read(length) if length else b"" + rendered = handle_request(self.store, method, self.path, body) + self.send_response(rendered.status_code) + self.send_header("content-type", rendered.content_type) + self.send_header("content-length", str(len(rendered.body))) + self.end_headers() + self.wfile.write(rendered.body) + + def do_GET(self) -> None: + self._dispatch("GET") + + def do_POST(self) -> None: + self._dispatch("POST") + + def do_DELETE(self) -> None: + self._dispatch("DELETE") + + + +DEFAULT_PORT: Final = 9100 + + +def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: + server = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) + sys.stderr.write(f"scripted-provider listening on http://{bind_host}:{port}\n") + server.serve_forever() + + +if __name__ == "__main__": + port_arg = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT + serve(port=port_arg) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py new file mode 100644 index 00000000000..8d7678cf9ca --- /dev/null +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -0,0 +1,115 @@ +"""Token-pricing e2e: every (frontier model, pricing-component case) cell runs a +scripted-usage call through a deployment registered on the cost-map proxy, and +the spend row plus response-cost header must equal literal arithmetic on the +test map's rates. + +Nothing here touches a real provider or the bundled cost map: the proxy's +upstream is the scripted-provider sidecar and its entire cost map is +tests/e2e/cost_map.json. +""" + +from __future__ import annotations + +import pytest + +from conftest import CostCalcClient, cost_rows, register_scenario_deployment +from cost_matrix import ( + FRONTIER_MODELS, + Case, + FrontierModel, + cases_for, + expected_cost, + expected_token_columns, +) +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatStreamOptions + +pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] + +_MATRIX: list[tuple[FrontierModel, Case]] = [ + (model, case) for model in FRONTIER_MODELS for case in cases_for(model) +] + + +def _case_id(param: tuple[FrontierModel, Case]) -> str: + model, case = param + return f"{model.map_key.replace('/', '-')}-{case.name}" + + +def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: + return ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"{marker} scripted pricing call")], + stream=case.stream, + stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, + service_tier=case.service_tier, + ) + + +class TestTokenPricing: + @pytest.mark.parametrize("model_case", _MATRIX, ids=_case_id) + @pytest.mark.covers("quota_management.spend_tracking.cost_matrix.logs_cost") + def test_scripted_usage_bills_at_map_rates( + self, + client: CostCalcClient, + resources: ResourceManager, + scoped_key: str, + model_case: tuple[FrontierModel, Case], + ) -> None: + model, case = model_case + marker = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=_chat_body(model_name, marker, case), + stream=case.stream, + ) + assert response.ok, ( + f"{model.map_key}/{case.name}: proxy returned {response.status_code}: {response.body[:400]}" + ) + assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" + + expected = expected_cost(model, case) + if case.exact_spend and not case.stream: + # Streamed responses commit headers before the bill is computed, so + # the x-litellm-response-cost header is asserted only on non-stream + # calls. + assert response.response_cost is not None and cost_rows.approx_equal( + response.response_cost, expected + ), ( + f"x-litellm-response-cost {response.response_cost} != expected {expected}" + ) + + row = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, f"no spend row with a cost breakdown landed for {model.map_key}/{case.name}" + + if not case.exact_spend and case.expect_zero_bill: + # The provider reported no usage and this wire has no proxy-side + # recount, so the bill is exactly zero. + assert row.spend is not None and row.spend == 0, f"no-usage stream billed {row.spend}: {row}" + return + if not case.exact_spend: + # stream_usage=absent: the provider reported no usage, so the row's + # token counts are the proxy's own recount; only assert a bill landed. + assert row.spend is not None and row.spend > 0, f"no-usage stream billed nothing: {row}" + return + + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected), ( + f"{model.map_key}/{case.name}: spend {row.spend} != expected {expected} " + f"(breakdown {row.breakdown.model_dump()})" + ) + + prompt_tokens, completion_tokens = expected_token_columns(model, case) + assert row.prompt_tokens == prompt_tokens, ( + f"prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + ) + assert row.completion_tokens == completion_tokens, ( + f"completion_tokens {row.completion_tokens} != {completion_tokens}" + ) + cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py new file mode 100644 index 00000000000..b1ef675d9ef --- /dev/null +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -0,0 +1,186 @@ +"""Wire-format e2e: one scripted upstream per provider wire, answering with a +usage payload where every token kind the wire can report is nonzero. The spend +row's gross input cost must equal fresh tokens at the input rate plus each cache +and audio component at its own rate -- proving the wire's usage shape landed the +cached tokens inside the total (OpenAI/Gemini) or as separate fields +(Anthropic), and that the biller subtracted them before billing fresh tokens. + +Also covers the Responses API wire (an openai/gpt-5.5-pro deployment bridged by +the proxy to POST /responses) and a streamed Anthropic-messages case. +""" + +from __future__ import annotations + +import pytest + +from conftest import CostCalcClient, cost_rows, register_scenario_deployment +from cost_matrix import ( + FRONTIER_MODELS, + Case, + FrontierModel, + expected_breakdown, + expected_token_columns, +) +from e2e_config import unique_marker +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatStreamOptions +from scripted_provider import ScriptedUsage + +pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] + +_MODELS: dict[str, FrontierModel] = {model.map_key: model for model in FRONTIER_MODELS} + +# One scripted usage per wire, every reportable token kind nonzero. +_WIRE_USAGE: dict[str, tuple[str, ScriptedUsage]] = { + "openai_chat": ( + "gpt-5.6", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "openai_responses": ( + "gpt-5.5-pro", + ScriptedUsage( + fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25, reasoning_tokens=15 + ), + ), + "anthropic_messages": ( + "claude-sonnet-5", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + ), + ), + "gemini_generate": ( + "gemini/gemini-3.8-flash", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "together_chat": ( + "together_ai/moonshotai/Kimi-K3", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "fireworks_chat": ( + "fireworks_ai/kimi-k3", + ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), + ), +} + + +class TestWireFormats: + @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) + @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") + def test_wire_usage_shape_bills_each_component( + self, + client: CostCalcClient, + resources: ResourceManager, + scoped_key: str, + wire: str, + ) -> None: + map_key, usage = _WIRE_USAGE[wire] + model = _MODELS[map_key] + case = Case(name="basic", usage=usage) + marker = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"{marker} scripted wire call")], + ), + ) + assert response.ok, f"{wire}: proxy returned {response.status_code}: {response.body[:400]}" + + expected = expected_breakdown(model, case) + row = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, f"{wire}: no spend row landed" + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( + f"{wire}: spend {row.spend} != expected {expected.total} " + f"(breakdown {row.breakdown.model_dump()})" + ) + breakdown = row.breakdown + assert breakdown.input_cost is not None and cost_rows.approx_equal( + breakdown.input_cost, expected.input_cost + ), ( + f"{wire}: gross input_cost {breakdown.input_cost} != expected {expected.input_cost}; " + "cached/written tokens billed at the input rate" + ) + assert breakdown.output_cost is not None and cost_rows.approx_equal( + breakdown.output_cost, expected.output_cost + ), f"{wire}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" + + prompt_tokens, completion_tokens = expected_token_columns(model, case) + assert row.prompt_tokens == prompt_tokens, ( + f"{wire}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + ) + assert row.completion_tokens == completion_tokens, ( + f"{wire}: completion_tokens {row.completion_tokens} != {completion_tokens}" + ) + cost_rows.assert_total_is_sum_of_components(row) + + @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") + def test_anthropic_streamed_usage_bills_each_component( + self, client: CostCalcClient, resources: ResourceManager, scoped_key: str + ) -> None: + map_key, usage = _WIRE_USAGE["anthropic_messages"] + model = _MODELS[map_key] + case = Case(name="stream", usage=usage, stream=True) + marker = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model_name, + messages=[ChatMessage(role="user", content=f"{marker} scripted anthropic stream")], + stream=True, + stream_options=ChatStreamOptions(include_usage=True), + ), + stream=True, + ) + assert response.ok, f"anthropic stream: proxy returned {response.status_code}: {response.body[:400]}" + assert response.stream_done, "anthropic stream did not reach its terminal event" + assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" + + expected = expected_breakdown(model, case) + row = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, "anthropic stream: no spend row landed" + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( + f"anthropic stream: spend {row.spend} != expected {expected.total} " + f"(breakdown {row.breakdown.model_dump()})" + ) + cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json new file mode 100644 index 00000000000..b761710bae3 --- /dev/null +++ b/tests/e2e/cost_map.json @@ -0,0 +1,352 @@ +{ + "claude-haiku-4-5": { + "cache_creation_input_token_cost": 0.00021, + "cache_creation_input_token_cost_above_1hr": 0.00028000000000000003, + "cache_read_input_token_cost": 7e-06, + "input_cost_per_token": 7.000000000000001e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00014000000000000001, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "claude-opus-5": { + "cache_creation_input_token_cost": 0.00015000000000000001, + "cache_creation_input_token_cost_above_1hr": 0.0002, + "cache_read_input_token_cost": 4.9999999999999996e-06, + "input_cost_per_token": 5e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.0001, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 0.00018, + "cache_creation_input_token_cost_above_1hr": 0.00024000000000000003, + "cache_read_input_token_cost": 6e-06, + "input_cost_per_token": 6.000000000000001e-05, + "litellm_provider": "anthropic", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00012000000000000002, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "fireworks_ai/deepseek-v4p1-flash": { + "cache_read_input_token_cost": 1.4e-05, + "input_cost_per_token": 0.00014000000000000001, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00028000000000000003, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "fireworks_ai/kimi-k3": { + "cache_read_input_token_cost": 1.2e-05, + "input_cost_per_token": 0.00012000000000000002, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00024000000000000003, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "fireworks_ai/qwen3p8-max": { + "cache_read_input_token_cost": 1.3e-05, + "input_cost_per_token": 0.00013000000000000002, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00026000000000000003, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gemini/gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 9e-06, + "input_cost_per_audio_token": 0.00054, + "input_cost_per_token": 9e-05, + "input_cost_per_token_above_200k_tokens": 0.00072, + "input_cost_per_token_flex": 0.000135, + "input_cost_per_token_priority": 0.000153, + "litellm_provider": "gemini", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0006299999999999999, + "output_cost_per_reasoning_token": 0.00045000000000000004, + "output_cost_per_token": 0.00018, + "output_cost_per_token_above_200k_tokens": 0.0008100000000000001, + "output_cost_per_token_flex": 0.00022500000000000002, + "output_cost_per_token_priority": 0.000243, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini/gemini-3.8-flash": { + "cache_read_input_token_cost": 8e-06, + "input_cost_per_audio_token": 0.00048, + "input_cost_per_token": 8e-05, + "input_cost_per_token_above_200k_tokens": 0.00064, + "input_cost_per_token_flex": 0.00012, + "input_cost_per_token_priority": 0.000136, + "litellm_provider": "gemini", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00056, + "output_cost_per_reasoning_token": 0.0004, + "output_cost_per_token": 0.00016, + "output_cost_per_token_above_200k_tokens": 0.00072, + "output_cost_per_token_flex": 0.0002, + "output_cost_per_token_priority": 0.000216, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gpt-5.3-codex": { + "cache_read_input_token_cost": 3e-06, + "input_cost_per_token": 3.0000000000000004e-05, + "input_cost_per_token_above_200k_tokens": 0.00024000000000000003, + "input_cost_per_token_flex": 4.5e-05, + "input_cost_per_token_priority": 5.1e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_reasoning_token": 0.00015000000000000001, + "output_cost_per_token": 6.000000000000001e-05, + "output_cost_per_token_above_200k_tokens": 0.00027, + "output_cost_per_token_flex": 7.500000000000001e-05, + "output_cost_per_token_priority": 8.099999999999999e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gpt-5.4-mini": { + "cache_creation_input_token_cost": 0.00012, + "cache_creation_input_token_cost_above_1hr": 0.00016, + "cache_read_input_token_cost": 4e-06, + "input_cost_per_audio_token": 0.00024, + "input_cost_per_token": 4e-05, + "input_cost_per_token_above_200k_tokens": 0.00032, + "input_cost_per_token_flex": 6e-05, + "input_cost_per_token_priority": 6.8e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00028, + "output_cost_per_reasoning_token": 0.0002, + "output_cost_per_token": 8e-05, + "output_cost_per_token_above_200k_tokens": 0.00036, + "output_cost_per_token_flex": 0.0001, + "output_cost_per_token_priority": 0.000108, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gpt-5.5-pro": { + "cache_read_input_token_cost": 2e-06, + "input_cost_per_token": 2e-05, + "input_cost_per_token_above_200k_tokens": 0.00016, + "input_cost_per_token_flex": 3e-05, + "input_cost_per_token_priority": 3.4e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "output_cost_per_reasoning_token": 0.0001, + "output_cost_per_token": 4e-05, + "output_cost_per_token_above_200k_tokens": 0.00018, + "output_cost_per_token_flex": 5e-05, + "output_cost_per_token_priority": 5.4e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gpt-5.6": { + "cache_creation_input_token_cost": 3e-05, + "cache_creation_input_token_cost_above_1hr": 4e-05, + "cache_read_input_token_cost": 1e-06, + "input_cost_per_audio_token": 6e-05, + "input_cost_per_token": 1e-05, + "input_cost_per_token_above_200k_tokens": 8e-05, + "input_cost_per_token_flex": 1.5e-05, + "input_cost_per_token_priority": 1.7e-05, + "litellm_provider": "openai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 7e-05, + "output_cost_per_reasoning_token": 5e-05, + "output_cost_per_token": 2e-05, + "output_cost_per_token_above_200k_tokens": 9e-05, + "output_cost_per_token_flex": 2.5e-05, + "output_cost_per_token_priority": 2.7e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "together_ai/moonshotai/Kimi-K3": { + "cache_creation_input_token_cost": 0.00030000000000000003, + "cache_creation_input_token_cost_above_1hr": 0.0004, + "cache_read_input_token_cost": 9.999999999999999e-06, + "input_cost_per_audio_token": 0.0006000000000000001, + "input_cost_per_token": 0.0001, + "input_cost_per_token_above_200k_tokens": 0.0008, + "input_cost_per_token_flex": 0.00015000000000000001, + "input_cost_per_token_priority": 0.00017, + "litellm_provider": "together_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0006999999999999999, + "output_cost_per_reasoning_token": 0.0005, + "output_cost_per_token": 0.0002, + "output_cost_per_token_above_200k_tokens": 0.0009000000000000001, + "output_cost_per_token_flex": 0.00025, + "output_cost_per_token_priority": 0.00027, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "together_ai/zai-org/GLM-5.3": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, + "cache_read_input_token_cost": 1.1e-05, + "input_cost_per_audio_token": 0.00066, + "input_cost_per_token": 0.00011, + "input_cost_per_token_above_200k_tokens": 0.00088, + "input_cost_per_token_flex": 0.000165, + "input_cost_per_token_priority": 0.000187, + "litellm_provider": "together_ai", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, + "output_cost_per_token": 0.00022, + "output_cost_per_token_above_200k_tokens": 0.00099, + "output_cost_per_token_flex": 0.000275, + "output_cost_per_token_priority": 0.000297, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + } +} diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index ad0914d455b..6b40e70125c 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -63,3 +63,5 @@ - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} - {id: quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [retrieve_batch_cost_joins_retrieving_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "The retrieve that first sees a batch in a terminal state prices it inline and writes its {provider_batch_id}_batch_cost row against the retrieving key, so the batch each run creates is one OpenAI fails at validation within seconds and the test retrieves it by its raw provider id with the same key until it is failed; a raw id is never owned by the CheckBatchCost poller, and the row must carry that key's token hash and alias"} - {id: quota_management.spend_tracking.key_attribution.poller_batch_cost_joins_creating_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [poller_batch_cost_joins_creating_key], exercised_on: [batches], source: "enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py", rationale: "The CheckBatchCost poller bills a completed, positive-cost batch created through a unified id against the key that created it, a different writer from the inline retrieve. No test claims this cell yet: OpenAI's completion window is 24h and both e2e stacks boot a fresh Postgres per build, so a completed batch is out of one run's reach and the managed list never shows an earlier run's batch; the cell stays visible as a gap until a run can hand a completed batch to the poller"} +- {id: quota_management.spend_tracking.cost_matrix.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_matrix, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "A scripted-usage call through the cost-map proxy bills every reported token kind at the deployment's test-map rate (input, output, cache read, 5m/1h cache write, reasoning, audio, above-threshold tiers, flex/priority service tiers, web search, response-model override) and lands on the row's cost_breakdown, streamed or not"} +- {id: quota_management.spend_tracking.scripted_wire.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: scripted_wire, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "Each provider wire shape (openai chat, responses, anthropic messages, gemini generateContent, together, fireworks) parses usage into the same spend components: the gross input cost is fresh tokens at the input rate plus each cache/audio component at its own rate, streamed anthropic included"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 896cb3e7efe..a891d9dcba2 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -145,6 +145,22 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK" REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" +# The cost_calculation suite needs a proxy booted with LITELLM_MODEL_COST_MAP_URL +# pointing at tests/e2e/cost_map.json (its whole map is test-owned rates) plus a +# scripted-provider sidecar; deselected unless the opt-in env var is set. +COST_MAP_OPT_IN_ENV = "E2E_COST_MAP_STACK" +# Base URL of the proxy running the test cost map. Defaults to the shared proxy +# so a local run only has to set the opt-in and boot the proxy accordingly. +COST_MAP_PROXY_URL = os.environ.get("E2E_COST_MAP_PROXY_URL", PROXY_BASE_URL).rstrip("/") +# Where the test runner reaches the scripted-provider sidecar's control API. +SCRIPTED_PROVIDER_CONTROL_URL = os.environ.get( + "E2E_SCRIPTED_PROVIDER_CONTROL_URL", "http://127.0.0.1:9100" +).rstrip("/") +# The api_base root deployments register with: how the proxy (possibly in +# another container) reaches the sidecar's provider wire. +SCRIPTED_PROVIDER_PROXY_BASE = os.environ.get( + "E2E_SCRIPTED_PROVIDER_PROXY_BASE", SCRIPTED_PROVIDER_CONTROL_URL +).rstrip("/") ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3")) diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index 1fdd3bd28ad..7d37bcc6d3e 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -11,3 +11,4 @@ markers = managed_files: needs a proxy running with require_managed_files enabled; deselected unless E2E_MANAGED_FILES_STACK is set prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set + cost_map_stack: needs a proxy whose whole cost map is tests/e2e/cost_map.json (LITELLM_MODEL_COST_MAP_URL) plus a scripted-provider sidecar; deselected unless E2E_COST_MAP_STACK is set From 269afbe382df06d33780571a40a55e527afea2b7 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 23:28:44 +0000 Subject: [PATCH 02/30] test(e2e): apply review nits to cost calculation suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/conftest.py | 28 +- tests/e2e/cost_calculation/cost_matrix.py | 174 ++-- tests/e2e/cost_calculation/scripted_client.py | 12 +- .../e2e/cost_calculation/scripted_provider.py | 803 +++++++++++------- .../test_token_pricing_e2e.py | 17 +- .../cost_calculation/test_wire_formats_e2e.py | 43 +- 6 files changed, 620 insertions(+), 457 deletions(-) diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 1bba3d50e1d..345ca26f7e3 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -13,7 +13,7 @@ from __future__ import annotations import importlib.util import sys -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path from types import ModuleType @@ -34,16 +34,16 @@ def _load_cost_rows() -> ModuleType: """Load quota_management/spend_tracking/cost_rows.py by path (the e2e tree has no package layout), the same trick the mcp suite uses for logging/datadog_reader.py.""" - path = ( + path: Final = ( Path(__file__).resolve().parent.parent / "quota_management" / "spend_tracking" / "cost_rows.py" ) - name = "e2e_spend_tracking_cost_rows" - spec = importlib.util.spec_from_file_location(name, path) + name: Final = "e2e_spend_tracking_cost_rows" + spec: Final = importlib.util.spec_from_file_location(name, path) assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) + module: Final = importlib.util.module_from_spec(spec) sys.modules[name] = module spec.loader.exec_module(module) return module @@ -59,7 +59,7 @@ class SpendCostBreakdown(Protocol): total_cost: float | None service_tier: str | None - def model_dump(self) -> dict[str, object]: ... + def model_dump(self) -> Mapping[str, object]: ... class SpendRowMetadata(Protocol): @@ -89,7 +89,9 @@ class CostRowsModule(Protocol): ] -cost_rows: Final[CostRowsModule] = cast(CostRowsModule, _load_cost_rows()) +cost_rows: Final[CostRowsModule] = cast( # cast-ok: cost_rows.py is loaded by path, so basedpyright has no importable name for it; its surface is declared in CostRowsModule + CostRowsModule, _load_cost_rows() +) @dataclass(frozen=True, slots=True) @@ -101,7 +103,7 @@ class CostCalcClient: @pytest.fixture(scope="session") def client() -> CostCalcClient: - proxy = build_proxy_client( + proxy: Final = build_proxy_client( base_url=COST_MAP_PROXY_URL, control_plane_base_url=COST_MAP_PROXY_URL, replica_urls=(COST_MAP_PROXY_URL,), @@ -118,18 +120,18 @@ def register_scenario_deployment( ) -> tuple[str, ScenarioHandle]: """Register the case's scenario on the sidecar plus a deployment pointed at it; both are torn down by ``resources``. Returns the callable model_name.""" - scenario: Scenario = case.scenario( + scenario: Final[Scenario] = case.scenario( scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" ) - handle = register_scenario(scenario) + handle: Final = register_scenario(scenario) resources.defer(lambda: delete_scenario(handle)) - model_name = f"{model.model_name}-{marker}" - model_id = client.proxy.register_model( + model_name: Final = f"{model.model_name}-{marker}" + model_id: Final = client.proxy.register_model( ModelNewBody( model_name=model_name, litellm_params=LiteLLMParamsBody( model=model.litellm_model, - api_key="sk-scripted-provider", + api_key=model.api_key, api_base=handle.api_base(), ), model_info=ModelInfoBody(), diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index bc466d7d823..e8b1d249559 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -18,9 +18,11 @@ creation), the case is absent from the matrix rather than silently zero. from __future__ import annotations import json +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path -from typing import Final, Literal +from types import MappingProxyType +from typing import Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, TypeAdapter @@ -64,8 +66,8 @@ class CostMapEntry(BaseModel): _COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) -_COST_MAP: Final[dict[str, CostMapEntry]] = _COST_MAP_ADAPTER.validate_python( - json.loads(COST_MAP_PATH.read_text()) +_COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( + _COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) ) TIER_THRESHOLD_TOKENS: Final = 200_000 @@ -109,7 +111,7 @@ class FrontierModel: # Response-model override targets: emit a sibling's bare provider-facing name so # the biller's provider-prefixed lookup lands on that sibling's map key. -_OVERRIDE_MODELS: Final[dict[str, str]] = { +_OVERRIDE_MODELS: Final[Mapping[str, str]] = MappingProxyType({ "gpt-5.6": "gpt-5.4-mini", "gpt-5.5-pro": "gpt-5.3-codex", "gpt-5.3-codex": "gpt-5.5-pro", @@ -124,9 +126,9 @@ _OVERRIDE_MODELS: Final[dict[str, str]] = { "fireworks_ai/kimi-k3": "qwen3p8-max", "fireworks_ai/qwen3p8-max": "kimi-k3", "fireworks_ai/deepseek-v4p1-flash": "kimi-k3", -} +}) -_OVERRIDE_MAP_KEYS: Final[dict[str, str]] = { +_OVERRIDE_MAP_KEYS: Final[Mapping[str, str]] = MappingProxyType({ "gpt-5.4-mini": "gpt-5.4-mini", "gpt-5.6": "gpt-5.6", "gpt-5.3-codex": "gpt-5.3-codex", @@ -139,7 +141,7 @@ _OVERRIDE_MAP_KEYS: Final[dict[str, str]] = { "moonshotai/Kimi-K3": "together_ai/moonshotai/Kimi-K3", "qwen3p8-max": "fireworks_ai/qwen3p8-max", "kimi-k3": "fireworks_ai/kimi-k3", -} +}) _FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( @@ -176,7 +178,7 @@ def _frontier() -> tuple[FrontierModel, ...]: FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() # Token kinds each wire can report, gating which pricing cases apply. -_WIRE_CAPS: Final[dict[str, frozenset[str]]] = { +_WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ "openai_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", @@ -205,9 +207,9 @@ _WIRE_CAPS: Final[dict[str, frozenset[str]]] = { "web_search", "response_model", "absent_usage", } ), -} +}) -CaseName = Literal[ +CaseName: TypeAlias = Literal[ "basic", "cache_read", "cache_write_5m", @@ -260,7 +262,7 @@ _BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) def _web_search_case(model: FrontierModel) -> Case: - counts_exactly = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") + counts_exactly: Final = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") return Case( name="web_search", usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), @@ -269,26 +271,24 @@ def _web_search_case(model: FrontierModel) -> Case: def cases_for(model: FrontierModel) -> tuple[Case, ...]: - rates = model.rates - caps = _WIRE_CAPS[model.wire] - cases: list[Case] = [Case(name="basic", usage=_BASIC_USAGE)] - if rates.cache_read_input_token_cost is not None and "cache_read" in caps: - cases.append( + rates: Final = model.rates + caps: Final = _WIRE_CAPS[model.wire] + candidates: Final[tuple[Case | None, ...]] = ( + Case(name="basic", usage=_BASIC_USAGE), + ( Case(name="cache_read", usage=ScriptedUsage(fresh_input_tokens=100, cache_read_tokens=50, output_tokens=30)) - ) - if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps: - cases.append( + if rates.cache_read_input_token_cost is not None and "cache_read" in caps + else None + ), + ( Case( name="cache_write_5m", usage=ScriptedUsage(fresh_input_tokens=90, cache_write_5m_tokens=60, output_tokens=30), ) - ) - if ( - rates.cache_creation_input_token_cost_above_1hr is not None - and rates.cache_creation_input_token_cost is not None - and "cache_write_1h" in caps - ): - cases.append( + if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps + else None + ), + ( Case( name="cache_write_1h", usage=ScriptedUsage( @@ -298,52 +298,61 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: output_tokens=30, ), ) - ) - if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps: - cases.append( + if ( + rates.cache_creation_input_token_cost_above_1hr is not None + and rates.cache_creation_input_token_cost is not None + and "cache_write_1h" in caps + ) + else None + ), + ( Case( name="reasoning", usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, reasoning_tokens=70), ) - ) - if ( - rates.input_cost_per_audio_token is not None - and rates.output_cost_per_audio_token is not None - and "audio" in caps - ): - cases.append( + if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps + else None + ), + ( Case( name="audio", usage=ScriptedUsage( fresh_input_tokens=100, audio_input_tokens=25, output_tokens=30, audio_output_tokens=15 ), ) - ) - if ( - rates.input_cost_per_token_above_200k_tokens is not None - and rates.output_cost_per_token_above_200k_tokens is not None - ): - cases.append( + if ( + rates.input_cost_per_audio_token is not None + and rates.output_cost_per_audio_token is not None + and "audio" in caps + ) + else None + ), + ( Case( name="tiered", usage=ScriptedUsage( fresh_input_tokens=TIER_THRESHOLD_TOKENS + 1, output_tokens=30 ), ) - ) - if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None: - cases.append( + if ( + rates.input_cost_per_token_above_200k_tokens is not None + and rates.output_cost_per_token_above_200k_tokens is not None + ) + else None + ), + ( Case(name="service_tier_flex", usage=_BASIC_USAGE, service_tier="flex") - ) - if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None: - cases.append( + if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None + else None + ), + ( Case(name="service_tier_priority", usage=_BASIC_USAGE, service_tier="priority") - ) - if rates.search_context_cost_per_query is not None and "web_search" in caps: - cases.append(_web_search_case(model)) - cases.append(Case(name="stream", usage=_BASIC_USAGE, stream=True)) - if "absent_usage" in caps: - cases.append( + if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None + else None + ), + _web_search_case(model) if rates.search_context_cost_per_query is not None and "web_search" in caps else None, + Case(name="stream", usage=_BASIC_USAGE, stream=True), + ( Case( name="stream_no_usage", usage=_BASIC_USAGE, @@ -355,10 +364,16 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: # wires recount tokens proxy-side and bill a nonzero amount. expect_zero_bill=model.wire == "openai_responses", ) - ) - if "response_model" in caps: - cases.append(Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True)) - return tuple(cases) + if "absent_usage" in caps + else None + ), + ( + Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True) + if "response_model" in caps + else None + ), + ) + return tuple(case for case in candidates if case is not None) @dataclass(frozen=True, slots=True) @@ -387,38 +402,41 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: to the tier's variants, falling back to the base rate when a variant is unset -- mirroring _get_token_base_cost in litellm's cost calculator. """ - rates = model.override_rates if case.response_model_override else model.rates - u = case.usage - prompt_tokens = ( + rates: Final = model.override_rates if case.response_model_override else model.rates + u: Final = case.usage + prompt_tokens: Final = ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens + u.audio_input_tokens ) - tiered = prompt_tokens > TIER_THRESHOLD_TOKENS - in_rate = rates.input_cost_per_token or 0.0 - out_rate = rates.output_cost_per_token or 0.0 - if case.service_tier == "flex": - in_rate = rates.input_cost_per_token_flex or in_rate - out_rate = rates.output_cost_per_token_flex or out_rate - if case.service_tier == "priority": - in_rate = rates.input_cost_per_token_priority or in_rate - out_rate = rates.output_cost_per_token_priority or out_rate - if tiered: - in_rate = rates.input_cost_per_token_above_200k_tokens or in_rate - out_rate = rates.output_cost_per_token_above_200k_tokens or out_rate - input_cost = ( + tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS + in_rate: Final = ( + (rates.input_cost_per_token_above_200k_tokens if tiered else None) + or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.input_cost_per_token + or 0.0 + ) + out_rate: Final = ( + (rates.output_cost_per_token_above_200k_tokens if tiered else None) + or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.output_cost_per_token + or 0.0 + ) + input_cost: Final = ( u.fresh_input_tokens * in_rate + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) ) - output_cost = ( + output_cost: Final = ( u.output_tokens * out_rate + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) ) - search = rates.search_context_cost_per_query - tool_cost = case.billed_web_search_calls * ( + search: Final = rates.search_context_cost_per_query + tool_cost: Final = case.billed_web_search_calls * ( search.search_context_size_medium if search and search.search_context_size_medium else 0.0 ) return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) @@ -432,7 +450,7 @@ def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: """(prompt_tokens, completion_tokens) the spend row should carry, per the wire's normalization: Anthropic folds cache read/write into prompt_tokens, everyone else reports the totals the wire emitted.""" - u = case.usage + u: Final = case.usage if model.wire == "anthropic_messages": return ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, diff --git a/tests/e2e/cost_calculation/scripted_client.py b/tests/e2e/cost_calculation/scripted_client.py index dceec02630a..9dbf9c98986 100644 --- a/tests/e2e/cost_calculation/scripted_client.py +++ b/tests/e2e/cost_calculation/scripted_client.py @@ -12,6 +12,7 @@ from e2e_config import SCRIPTED_PROVIDER_CONTROL_URL, SCRIPTED_PROVIDER_PROXY_BA from e2e_http import URL, NoBody, unwrap, post from e2e_http import delete as http_delete from scripted_provider import ( + WIRE_MOUNTS, Scenario, ScenarioDeleted, ScenarioRegistered, @@ -29,19 +30,12 @@ class ScenarioHandle: return f"{self.proxy_base}/{self.scenario_id}/{self._mount()}" def _mount(self) -> str: - return { - "openai_chat": "openai", - "openai_responses": "openai", - "anthropic_messages": "anthropic", - "gemini_generate": "gemini", - "together_chat": "together", - "fireworks_chat": "fireworks", - }[self.wire] + return WIRE_MOUNTS[self.wire] def register_scenario(scenario: Scenario) -> ScenarioHandle: """POST the scenario to the sidecar's control API and return its handle.""" - result = unwrap( + result: Final = unwrap( post( URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios"), headers=NoBody(), diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 93a6f49ec25..f1deafd1bc5 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -31,14 +31,16 @@ import json import sys import threading import time +from collections.abc import Mapping from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Final, Literal +from types import MappingProxyType +from typing import Final, Literal, TypeAlias from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError -Wire = Literal[ +Wire: TypeAlias = Literal[ "openai_chat", "openai_responses", "anthropic_messages", @@ -47,17 +49,19 @@ Wire = Literal[ "fireworks_chat", ] -_WIRE_MOUNTS: Final[dict[str, str]] = { - "openai_chat": "openai", - "openai_responses": "openai", - "anthropic_messages": "anthropic", - "gemini_generate": "gemini", - "together_chat": "together", - "fireworks_chat": "fireworks", -} +WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( + { + "openai_chat": "openai", + "openai_responses": "openai", + "anthropic_messages": "anthropic", + "gemini_generate": "gemini", + "together_chat": "together", + "fireworks_chat": "fireworks", + } +) -StreamUsage = Literal["final_chunk", "absent"] -ServiceTier = Literal["flex", "priority"] +StreamUsage: TypeAlias = Literal["final_chunk", "absent"] +ServiceTier: TypeAlias = Literal["flex", "priority"] class ScriptedUsage(BaseModel): @@ -106,7 +110,7 @@ class Scenario(BaseModel): @property def mount(self) -> str: - return _WIRE_MOUNTS[self.wire] + return WIRE_MOUNTS[self.wire] class ScenarioRegistered(BaseModel): @@ -128,369 +132,494 @@ class RenderedResponse: body: bytes -def _json_bytes(payload: dict[str, object]) -> bytes: - return json.dumps(payload).encode("utf-8") +def _jobj(*pairs: tuple[str, object]) -> Mapping[str, object]: + """A JSON object payload built in one shot and frozen.""" + return MappingProxyType(dict(pairs)) -def _sse(events: tuple[tuple[str | None, dict[str, object] | str], ...]) -> bytes: - frames: list[str] = [] - for event_name, data in events: - head = f"event: {event_name}\n" if event_name is not None else "" - payload = data if isinstance(data, str) else json.dumps(data) - frames.append(f"{head}data: {payload}\n\n") - return "".join(frames).encode("utf-8") +def _jobj_opt(*pairs: tuple[str, object] | None) -> Mapping[str, object]: + """``_jobj`` where a ``None`` pair means the field is absent.""" + return MappingProxyType(dict(pair for pair in pairs if pair is not None)) + + +def _json_bytes(payload: Mapping[str, object]) -> bytes: + return json.dumps(payload, default=dict).encode("utf-8") + + +def _sse_frame(event_name: str | None, data: Mapping[str, object] | str) -> str: + head: Final = f"event: {event_name}\n" if event_name is not None else "" + payload: Final = data if isinstance(data, str) else json.dumps(data, default=dict) + return f"{head}data: {payload}\n\n" + + +def _sse(events: tuple[tuple[str | None, Mapping[str, object] | str], ...]) -> bytes: + return "".join(_sse_frame(event_name, data) for event_name, data in events).encode("utf-8") # ---------- per-wire usage shapes ---------- -def _openai_usage(u: ScriptedUsage) -> dict[str, object]: - prompt_tokens = ( +def _openai_usage(u: ScriptedUsage) -> Mapping[str, object]: + prompt_tokens: Final = ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens + u.audio_input_tokens ) - completion_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - prompt_details: dict[str, object] = {} - if u.cache_read_tokens: - prompt_details["cached_tokens"] = u.cache_read_tokens - if u.cache_write_5m_tokens or u.cache_write_1h_tokens: - prompt_details["cache_write_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens - prompt_details["cache_creation_token_details"] = { - "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, - "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, - } - if u.audio_input_tokens: - prompt_details["audio_tokens"] = u.audio_input_tokens - completion_details: dict[str, object] = {} - if u.reasoning_tokens: - completion_details["reasoning_tokens"] = u.reasoning_tokens - if u.audio_output_tokens: - completion_details["audio_tokens"] = u.audio_output_tokens - usage: dict[str, object] = { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens, - } - if prompt_details: - usage["prompt_tokens_details"] = prompt_details - if completion_details: - usage["completion_tokens_details"] = completion_details - return usage + completion_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + prompt_details: Final = _jobj_opt( + ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ( + ("cache_write_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ( + ( + "cache_creation_token_details", + _jobj( + ("ephemeral_5m_input_tokens", u.cache_write_5m_tokens), + ("ephemeral_1h_input_tokens", u.cache_write_1h_tokens), + ), + ) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ("audio_tokens", u.audio_input_tokens) if u.audio_input_tokens else None, + ) + completion_details: Final = _jobj_opt( + ("reasoning_tokens", u.reasoning_tokens) if u.reasoning_tokens else None, + ("audio_tokens", u.audio_output_tokens) if u.audio_output_tokens else None, + ) + return _jobj_opt( + ("prompt_tokens", prompt_tokens), + ("completion_tokens", completion_tokens), + ("total_tokens", prompt_tokens + completion_tokens), + ("prompt_tokens_details", prompt_details) if prompt_details else None, + ("completion_tokens_details", completion_details) if completion_details else None, + ) -def _anthropic_usage(u: ScriptedUsage) -> dict[str, object]: +def _anthropic_usage(u: ScriptedUsage) -> Mapping[str, object]: # Anthropic reports uncached-only input_tokens; cache reads and writes ride # top-level fields, with the 5m/1h write split under cache_creation. - usage: dict[str, object] = { - "input_tokens": u.fresh_input_tokens, - "output_tokens": u.output_tokens, - } - if u.cache_read_tokens: - usage["cache_read_input_tokens"] = u.cache_read_tokens - if u.cache_write_5m_tokens or u.cache_write_1h_tokens: - usage["cache_creation_input_tokens"] = u.cache_write_5m_tokens + u.cache_write_1h_tokens - usage["cache_creation"] = { - "ephemeral_5m_input_tokens": u.cache_write_5m_tokens, - "ephemeral_1h_input_tokens": u.cache_write_1h_tokens, - } - if u.web_search_calls: - usage["server_tool_use"] = {"web_search_requests": u.web_search_calls} - return usage + return _jobj_opt( + ("input_tokens", u.fresh_input_tokens), + ("output_tokens", u.output_tokens), + ("cache_read_input_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ( + ("cache_creation_input_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ( + ( + "cache_creation", + _jobj( + ("ephemeral_5m_input_tokens", u.cache_write_5m_tokens), + ("ephemeral_1h_input_tokens", u.cache_write_1h_tokens), + ), + ) + if u.cache_write_5m_tokens or u.cache_write_1h_tokens + else None + ), + ( + ("server_tool_use", _jobj(("web_search_requests", u.web_search_calls))) + if u.web_search_calls + else None + ), + ) -def _gemini_usage(u: ScriptedUsage) -> dict[str, object]: +def _gemini_usage(u: ScriptedUsage) -> Mapping[str, object]: # promptTokenCount carries the cached count inside it; TEXT modality is the # cached-inclusive text count so litellm's implicit-caching subtraction lands # on the fresh figure. candidatesTokenCount includes reasoning + audio. - prompt_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - candidates = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - usage: dict[str, object] = { - "promptTokenCount": prompt_tokens, - "candidatesTokenCount": candidates, - "totalTokenCount": prompt_tokens + candidates, - } - if u.cache_read_tokens: - usage["cachedContentTokenCount"] = u.cache_read_tokens - if u.reasoning_tokens: - usage["thoughtsTokenCount"] = u.reasoning_tokens - prompt_details = [{"modality": "TEXT", "tokenCount": u.fresh_input_tokens + u.cache_read_tokens}] - if u.audio_input_tokens: - prompt_details.append({"modality": "AUDIO", "tokenCount": u.audio_input_tokens}) - usage["promptTokensDetails"] = prompt_details - if u.audio_output_tokens: - usage["candidatesTokensDetails"] = [ - {"modality": "TEXT", "tokenCount": u.output_tokens + u.reasoning_tokens}, - {"modality": "AUDIO", "tokenCount": u.audio_output_tokens}, - ] - return usage + prompt_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + candidates: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + return _jobj_opt( + ("promptTokenCount", prompt_tokens), + ("candidatesTokenCount", candidates), + ("totalTokenCount", prompt_tokens + candidates), + ("cachedContentTokenCount", u.cache_read_tokens) if u.cache_read_tokens else None, + ("thoughtsTokenCount", u.reasoning_tokens) if u.reasoning_tokens else None, + ( + "promptTokensDetails", + ( + _jobj(("modality", "TEXT"), ("tokenCount", u.fresh_input_tokens + u.cache_read_tokens)), + *( + (_jobj(("modality", "AUDIO"), ("tokenCount", u.audio_input_tokens)),) + if u.audio_input_tokens + else () + ), + ), + ), + ( + ( + "candidatesTokensDetails", + ( + _jobj(("modality", "TEXT"), ("tokenCount", u.output_tokens + u.reasoning_tokens)), + _jobj(("modality", "AUDIO"), ("tokenCount", u.audio_output_tokens)), + ), + ) + if u.audio_output_tokens + else None + ), + ) -def _responses_usage(u: ScriptedUsage) -> dict[str, object]: - input_tokens = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - output_tokens = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - usage: dict[str, object] = { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - "total_tokens": input_tokens + output_tokens, - } - input_details: dict[str, object] = {} - if u.cache_read_tokens: - input_details["cached_tokens"] = u.cache_read_tokens - if input_details: - usage["input_tokens_details"] = input_details - if u.reasoning_tokens: - usage["output_tokens_details"] = {"reasoning_tokens": u.reasoning_tokens} - return usage +def _responses_usage(u: ScriptedUsage) -> Mapping[str, object]: + input_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + output_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens + input_details: Final = _jobj_opt( + ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ) + return _jobj_opt( + ("input_tokens", input_tokens), + ("output_tokens", output_tokens), + ("total_tokens", input_tokens + output_tokens), + ("input_tokens_details", input_details) if input_details else None, + ( + ("output_tokens_details", _jobj(("reasoning_tokens", u.reasoning_tokens))) + if u.reasoning_tokens + else None + ), + ) # ---------- per-wire responses ---------- -def _openai_message(scenario: Scenario) -> dict[str, object]: - message: dict[str, object] = {"role": "assistant", "content": scenario.output.text} - if scenario.usage.web_search_calls: - message["annotations"] = [ - { - "type": "url_citation", - "url_citation": { - "url": "https://scripted.example/source", - "title": "scripted source", - "start_index": 0, - "end_index": 1, - }, - } - for _ in range(scenario.usage.web_search_calls) - ] - return message +def _openai_message(scenario: Scenario) -> Mapping[str, object]: + return _jobj_opt( + ("role", "assistant"), + ("content", scenario.output.text), + ( + ( + "annotations", + tuple( + _jobj( + ("type", "url_citation"), + ( + "url_citation", + _jobj( + ("url", "https://scripted.example/source"), + ("title", "scripted source"), + ("start_index", 0), + ("end_index", 1), + ), + ), + ) + for _ in range(scenario.usage.web_search_calls) + ), + ) + if scenario.usage.web_search_calls + else None + ), + ) -def _openai_chat_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - body: dict[str, object] = { - "id": f"chatcmpl-{scenario.scenario_id}", - "object": "chat.completion", - "created": int(time.time()), - "model": scenario.output.response_model or requested_model, - "choices": [ - { - "index": 0, - "message": _openai_message(scenario), - "finish_reason": scenario.output.finish_reason, - } - ], - "usage": _openai_usage(scenario.usage), - } - if scenario.service_tier is not None: - body["service_tier"] = scenario.service_tier - if scenario.output.provider_cost is not None: - body["cost"] = scenario.output.provider_cost - return body +def _openai_chat_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj_opt( + ("id", f"chatcmpl-{scenario.scenario_id}"), + ("object", "chat.completion"), + ("created", int(time.time())), + ("model", scenario.output.response_model or requested_model), + ( + "choices", + ( + _jobj( + ("index", 0), + ("message", _openai_message(scenario)), + ("finish_reason", scenario.output.finish_reason), + ), + ), + ), + ("usage", _openai_usage(scenario.usage)), + ("service_tier", scenario.service_tier) if scenario.service_tier is not None else None, + ("cost", scenario.output.provider_cost) if scenario.output.provider_cost is not None else None, + ) -def _openai_chunk(scenario: Scenario, requested_model: str, **kw: object) -> dict[str, object]: - chunk: dict[str, object] = { - "id": f"chatcmpl-{scenario.scenario_id}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": scenario.output.response_model or requested_model, - } - chunk.update(kw) - return chunk +def _openai_chunk( + scenario: Scenario, + requested_model: str, + choices: tuple[Mapping[str, object], ...] = (), + usage: Mapping[str, object] | None = None, +) -> Mapping[str, object]: + return _jobj_opt( + ("id", f"chatcmpl-{scenario.scenario_id}"), + ("object", "chat.completion.chunk"), + ("created", int(time.time())), + ("model", scenario.output.response_model or requested_model), + ("choices", choices), + ("usage", usage), + ) def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: - _EMPTY_DELTA: Final[dict[str, object]] = {} - delta: dict[str, object] = {"role": "assistant", "content": scenario.output.text} - if scenario.usage.web_search_calls: - delta["annotations"] = _openai_message(scenario)["annotations"] - events: list[tuple[str | None, dict[str, object] | str]] = [ + delta: Final = _jobj_opt( + ("role", "assistant"), + ("content", scenario.output.text), ( - None, - _openai_chunk( - scenario, - requested_model, - choices=[{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], - ), + ("annotations", _openai_message(scenario)["annotations"]) + if scenario.usage.web_search_calls + else None ), + ) + return _sse( ( - None, - _openai_chunk( - scenario, - requested_model, - choices=[{"index": 0, "delta": delta, "finish_reason": None}], + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=(_jobj(("index", 0), ("delta", _jobj(("role", "assistant"))), ("finish_reason", None)),), + ), ), - ), - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=[ - { - "index": 0, - "delta": _EMPTY_DELTA, - "finish_reason": scenario.output.finish_reason, - } - ], + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=(_jobj(("index", 0), ("delta", delta), ("finish_reason", None)),), + ), ), - ), - ] - if scenario.stream_usage == "final_chunk": - events.append( - (None, _openai_chunk(scenario, requested_model, choices=(), usage=_openai_usage(scenario.usage))) + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=( + _jobj( + ("index", 0), + ("delta", _jobj()), + ("finish_reason", scenario.output.finish_reason), + ), + ), + ), + ), + *( + ((None, _openai_chunk(scenario, requested_model, usage=_openai_usage(scenario.usage))),) + if scenario.stream_usage == "final_chunk" + else () + ), + (None, "[DONE]"), ) - events.append((None, "[DONE]")) - return _sse(tuple(events)) + ) -def _anthropic_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - return { - "id": f"msg_{scenario.scenario_id}", - "type": "message", - "role": "assistant", - "model": scenario.output.response_model or requested_model, - "content": [{"type": "text", "text": scenario.output.text}], - "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, - "usage": _anthropic_usage(scenario.usage), - } +def _anthropic_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ("id", f"msg_{scenario.scenario_id}"), + ("type", "message"), + ("role", "assistant"), + ("model", scenario.output.response_model or requested_model), + ("content", (_jobj(("type", "text"), ("text", scenario.output.text)),)), + ( + "stop_reason", + "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, + ), + ("usage", _anthropic_usage(scenario.usage)), + ) def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: - emit_usage = scenario.stream_usage == "final_chunk" - input_usage = {k: v for k, v in _anthropic_usage(scenario.usage).items() if k != "output_tokens"} - message_start: dict[str, object] = { - "type": "message_start", - "message": { - "id": f"msg_{scenario.scenario_id}", - "type": "message", - "role": "assistant", - "model": scenario.output.response_model or requested_model, - "content": [], - "stop_reason": None, - **({"usage": input_usage} if emit_usage else {}), - }, - } - message_delta: dict[str, object] = { - "type": "message_delta", - "delta": { - "stop_reason": "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason - }, - **({"usage": {"output_tokens": scenario.usage.output_tokens}} if emit_usage else {}), - } + emit_usage: Final = scenario.stream_usage == "final_chunk" + input_usage: Final = _jobj( + *( + (key, value) + for key, value in _anthropic_usage(scenario.usage).items() + if key != "output_tokens" + ) + ) + message_start: Final = _jobj( + ("type", "message_start"), + ( + "message", + _jobj_opt( + ("id", f"msg_{scenario.scenario_id}"), + ("type", "message"), + ("role", "assistant"), + ("model", scenario.output.response_model or requested_model), + ("content", ()), + ("stop_reason", None), + ("usage", input_usage) if emit_usage else None, + ), + ), + ) + message_delta: Final = _jobj_opt( + ("type", "message_delta"), + ( + "delta", + _jobj( + ( + "stop_reason", + "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, + ) + ), + ), + ( + ("usage", _jobj(("output_tokens", scenario.usage.output_tokens))) + if emit_usage + else None + ), + ) return _sse( ( ("message_start", message_start), ( "content_block_start", - { - "type": "content_block_start", - "index": 0, - "content_block": {"type": "text", "text": ""}, - }, + _jobj( + ("type", "content_block_start"), + ("index", 0), + ("content_block", _jobj(("type", "text"), ("text", ""))), + ), ), ( "content_block_delta", - { - "type": "content_block_delta", - "index": 0, - "delta": {"type": "text_delta", "text": scenario.output.text}, - }, + _jobj( + ("type", "content_block_delta"), + ("index", 0), + ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), + ), ), - ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("content_block_stop", _jobj(("type", "content_block_stop"), ("index", 0))), ("message_delta", message_delta), - ("message_stop", {"type": "message_stop"}), + ("message_stop", _jobj(("type", "message_stop"))), ) ) -def _gemini_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - candidate: dict[str, object] = { - "content": {"parts": [{"text": scenario.output.text}], "role": "model"}, - "finishReason": "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), - "index": 0, - } - if scenario.usage.web_search_calls: - candidate["groundingMetadata"] = { - "webSearchQueries": [f"query {i}" for i in range(scenario.usage.web_search_calls)] - } - return { - "candidates": [candidate], - "usageMetadata": _gemini_usage(scenario.usage), - "modelVersion": scenario.output.response_model or requested_model, - } +def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ( + "candidates", + ( + _jobj_opt( + ( + "content", + _jobj( + ("parts", (_jobj(("text", scenario.output.text)),)), + ("role", "model"), + ), + ), + ( + "finishReason", + "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), + ), + ("index", 0), + ( + ( + "groundingMetadata", + _jobj( + ( + "webSearchQueries", + tuple(f"query {i}" for i in range(scenario.usage.web_search_calls)), + ) + ), + ) + if scenario.usage.web_search_calls + else None + ), + ), + ), + ), + ("usageMetadata", _gemini_usage(scenario.usage)), + ("modelVersion", scenario.output.response_model or requested_model), + ) def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: - first = _gemini_body(scenario, requested_model) - if scenario.stream_usage == "absent": - first = {k: v for k, v in first.items() if k != "usageMetadata"} - events: list[tuple[str | None, dict[str, object] | str]] = [(None, first)] - if scenario.stream_usage == "final_chunk": - events.append( - ( - None, - { - "candidates": [], - "usageMetadata": _gemini_usage(scenario.usage), - "modelVersion": scenario.output.response_model or requested_model, - }, - ) - ) - return _sse(tuple(events)) - - -def _responses_body(scenario: Scenario, requested_model: str) -> dict[str, object]: - output: list[dict[str, object]] = [ - {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} - for i in range(scenario.usage.web_search_calls) - ] - output.append( - { - "type": "message", - "id": f"msg_{scenario.scenario_id}", - "status": "completed", - "role": "assistant", - "content": [ - { - "type": "output_text", - "text": scenario.output.text, - "annotations": [], - } - ], - } + emit_usage: Final = scenario.stream_usage == "final_chunk" + first: Final = ( + _jobj(*((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata")) + if scenario.stream_usage == "absent" + else _gemini_body(scenario, requested_model) + ) + return _sse( + ( + (None, first), + *( + ( + ( + None, + _jobj( + ("candidates", ()), + ("usageMetadata", _gemini_usage(scenario.usage)), + ("modelVersion", scenario.output.response_model or requested_model), + ), + ), + ) + if emit_usage + else () + ), + ) + ) + + +def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ("id", f"resp_{scenario.scenario_id}"), + ("object", "response"), + ("created_at", int(time.time())), + ("status", "completed"), + ("model", scenario.output.response_model or requested_model), + ( + "output", + ( + *( + _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) + for i in range(scenario.usage.web_search_calls) + ), + _jobj( + ("type", "message"), + ("id", f"msg_{scenario.scenario_id}"), + ("status", "completed"), + ("role", "assistant"), + ( + "content", + ( + _jobj( + ("type", "output_text"), + ("text", scenario.output.text), + ("annotations", ()), + ), + ), + ), + ), + ), + ), + ("usage", _responses_usage(scenario.usage)), ) - return { - "id": f"resp_{scenario.scenario_id}", - "object": "response", - "created_at": int(time.time()), - "status": "completed", - "model": scenario.output.response_model or requested_model, - "output": output, - "usage": _responses_usage(scenario.usage), - } def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: - completed = _responses_body(scenario, requested_model) - if scenario.stream_usage == "absent": - completed = {k: v for k, v in completed.items() if k != "usage"} - created = {**completed, "status": "in_progress", "usage": None} + completed: Final = ( + _jobj(*((key, value) for key, value in _responses_body(scenario, requested_model).items() if key != "usage")) + if scenario.stream_usage == "absent" + else _responses_body(scenario, requested_model) + ) + created: Final = _jobj( + *((key, value) for key, value in completed.items() if key not in ("status", "usage")), + ("status", "in_progress"), + ("usage", None), + ) return _sse( ( - ("response.created", {"type": "response.created", "response": created}), + ("response.created", _jobj(("type", "response.created"), ("response", created))), ( "response.output_text.delta", - { - "type": "response.output_text.delta", - "item_id": f"msg_{scenario.scenario_id}", - "output_index": scenario.usage.web_search_calls, - "content_index": 0, - "delta": scenario.output.text, - }, + _jobj( + ("type", "response.output_text.delta"), + ("item_id", f"msg_{scenario.scenario_id}"), + ("output_index", scenario.usage.web_search_calls), + ("content_index", 0), + ("delta", scenario.output.text), + ), ), - ("response.completed", {"type": "response.completed", "response": completed}), + ("response.completed", _jobj(("type", "response.completed"), ("response", completed))), ) ) @@ -538,11 +667,11 @@ class _ScenarioStore: _REQUEST_BODY: Final = TypeAdapter(dict[str, object]) -def _request_body(body: bytes) -> dict[str, object]: +def _request_body(body: bytes) -> Mapping[str, object]: try: return _REQUEST_BODY.validate_json(body) except ValueError: - return {} + return MappingProxyType({}) def _request_wants_stream(path_tail: str, body: bytes) -> bool: @@ -554,52 +683,66 @@ def _request_wants_stream(path_tail: str, body: bytes) -> bool: def _request_model(body: bytes) -> str: - model = _request_body(body).get("model") + model: Final = _request_body(body).get("model") return model if isinstance(model, str) else "unknown" def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: - path = urlsplit(raw_path).path - segments = [segment for segment in path.split("/") if segment] - if method == "GET" and segments == ["health"]: - return RenderedResponse(200, "application/json", _json_bytes({"status": "ok"})) + path: Final = urlsplit(raw_path).path + segments: Final = tuple(segment for segment in path.split("/") if segment) + if method == "GET" and segments == ("health",): + return RenderedResponse(200, "application/json", _json_bytes(_jobj(("status", "ok")))) if segments and segments[0] == "_scenarios": if method == "POST" and len(segments) == 1: try: - scenario = Scenario.model_validate_json(body) + scenario: Final = Scenario.model_validate_json(body) except ValidationError as exc: - return RenderedResponse(400, "application/json", _json_bytes({"error": str(exc)})) + return RenderedResponse( + 400, "application/json", _json_bytes(_jobj(("error", str(exc)))) + ) store.put(scenario) - return RenderedResponse(200, "application/json", _json_bytes({"scenario_id": scenario.scenario_id})) - if method == "DELETE" and len(segments) == 2: - deleted = store.drop(segments[1]) return RenderedResponse( - 200 if deleted else 404, "application/json", _json_bytes({"deleted": deleted}) + 200, "application/json", _json_bytes(_jobj(("scenario_id", scenario.scenario_id))) ) - return RenderedResponse(404, "application/json", _json_bytes({"error": "unknown control route"})) + if method == "DELETE" and len(segments) == 2: + deleted: Final = store.drop(segments[1]) + return RenderedResponse( + 200 if deleted else 404, + "application/json", + _json_bytes(_jobj(("deleted", deleted))), + ) + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", "unknown control route"))) + ) if len(segments) < 2 or method != "POST": - return RenderedResponse(404, "application/json", _json_bytes({"error": f"no route for {method} {path}"})) + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) + ) scenario_id, mount = segments[0], segments[1] - scenario = store.get(scenario_id) - if scenario is None: - return RenderedResponse(404, "application/json", _json_bytes({"error": f"unknown scenario {scenario_id}"})) - if scenario.mount != mount: + found: Final = store.get(scenario_id) + if found is None: + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", f"unknown scenario {scenario_id}"))) + ) + if found.mount != mount: return RenderedResponse( 400, "application/json", - _json_bytes({"error": f"scenario {scenario_id} is wire {scenario.wire}, not mount {mount}"}), + _json_bytes( + _jobj(("error", f"scenario {scenario_id} is wire {found.wire}, not mount {mount}")) + ), ) - tail = "/".join(segments[2:]) - return _render(scenario, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) + tail: Final = "/".join(segments[2:]) + return _render(found, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) class _ScriptedHandler(BaseHTTPRequestHandler): store: Final[_ScenarioStore] = _ScenarioStore() def _dispatch(self, method: str) -> None: - length = int(self.headers.get("content-length") or 0) - body = self.rfile.read(length) if length else b"" - rendered = handle_request(self.store, method, self.path, body) + length: Final = int(self.headers.get("content-length") or 0) + body: Final = self.rfile.read(length) if length else b"" + rendered: Final = handle_request(self.store, method, self.path, body) self.send_response(rendered.status_code) self.send_header("content-type", rendered.content_type) self.send_header("content-length", str(len(rendered.body))) @@ -621,11 +764,11 @@ DEFAULT_PORT: Final = 9100 def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: - server = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) + server: Final = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) sys.stderr.write(f"scripted-provider listening on http://{bind_host}:{port}\n") server.serve_forever() if __name__ == "__main__": - port_arg = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT + port_arg: Final = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT serve(port=port_arg) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 8d7678cf9ca..e210dad94b1 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -11,6 +11,7 @@ tests/e2e/cost_map.json. from __future__ import annotations import pytest +from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( @@ -25,11 +26,11 @@ from e2e_config import unique_marker from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatStreamOptions -pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] +pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark -_MATRIX: list[tuple[FrontierModel, Case]] = [ +_MATRIX: Final[tuple[tuple[FrontierModel, Case], ...]] = tuple( (model, case) for model in FRONTIER_MODELS for case in cases_for(model) -] +) def _case_id(param: tuple[FrontierModel, Case]) -> str: @@ -40,7 +41,7 @@ def _case_id(param: tuple[FrontierModel, Case]) -> str: def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: return ChatBody( model=model_name, - messages=[ChatMessage(role="user", content=f"{marker} scripted pricing call")], + messages=(ChatMessage(role="user", content=f"{marker} scripted pricing call"),), stream=case.stream, stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, service_tier=case.service_tier, @@ -58,9 +59,9 @@ class TestTokenPricing: model_case: tuple[FrontierModel, Case], ) -> None: model, case = model_case - marker = unique_marker() + marker: Final = unique_marker() model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response = client.proxy.transport.send( + response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), json=_chat_body(model_name, marker, case), @@ -71,7 +72,7 @@ class TestTokenPricing: ) assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - expected = expected_cost(model, case) + expected: Final = expected_cost(model, case) if case.exact_spend and not case.stream: # Streamed responses commit headers before the bill is computed, so # the x-litellm-response-cost header is asserted only on non-stream @@ -82,7 +83,7 @@ class TestTokenPricing: f"x-litellm-response-cost {response.response_cost} != expected {expected}" ) - row = cost_rows.poll_cost_row_where( + row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index b1ef675d9ef..c0276cf370c 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -12,6 +12,9 @@ the proxy to POST /responses) and a streamed Anthropic-messages case. from __future__ import annotations import pytest +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( @@ -26,12 +29,14 @@ from lifecycle import ResourceManager from models import ChatBody, ChatMessage, ChatStreamOptions from scripted_provider import ScriptedUsage -pytestmark = [pytest.mark.e2e, pytest.mark.cost_map_stack] +pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark -_MODELS: dict[str, FrontierModel] = {model.map_key: model for model in FRONTIER_MODELS} +_MODELS: Final[Mapping[str, FrontierModel]] = MappingProxyType( + {model.map_key: model for model in FRONTIER_MODELS} +) # One scripted usage per wire, every reportable token kind nonzero. -_WIRE_USAGE: dict[str, tuple[str, ScriptedUsage]] = { +_WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ "openai_chat": ( "gpt-5.6", ScriptedUsage( @@ -89,7 +94,7 @@ _WIRE_USAGE: dict[str, tuple[str, ScriptedUsage]] = { "fireworks_ai/kimi-k3", ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), ), -} +}) class TestWireFormats: @@ -103,22 +108,22 @@ class TestWireFormats: wire: str, ) -> None: map_key, usage = _WIRE_USAGE[wire] - model = _MODELS[map_key] - case = Case(name="basic", usage=usage) - marker = unique_marker() + model: Final = _MODELS[map_key] + case: Final = Case(name="basic", usage=usage) + marker: Final = unique_marker() model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response = client.proxy.transport.send( + response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), json=ChatBody( model=model_name, - messages=[ChatMessage(role="user", content=f"{marker} scripted wire call")], + messages=(ChatMessage(role="user", content=f"{marker} scripted wire call"),), ), ) assert response.ok, f"{wire}: proxy returned {response.status_code}: {response.body[:400]}" - expected = expected_breakdown(model, case) - row = cost_rows.poll_cost_row_where( + expected: Final = expected_breakdown(model, case) + row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, @@ -128,7 +133,7 @@ class TestWireFormats: f"{wire}: spend {row.spend} != expected {expected.total} " f"(breakdown {row.breakdown.model_dump()})" ) - breakdown = row.breakdown + breakdown: Final = row.breakdown assert breakdown.input_cost is not None and cost_rows.approx_equal( breakdown.input_cost, expected.input_cost ), ( @@ -153,16 +158,16 @@ class TestWireFormats: self, client: CostCalcClient, resources: ResourceManager, scoped_key: str ) -> None: map_key, usage = _WIRE_USAGE["anthropic_messages"] - model = _MODELS[map_key] - case = Case(name="stream", usage=usage, stream=True) - marker = unique_marker() + model: Final = _MODELS[map_key] + case: Final = Case(name="stream", usage=usage, stream=True) + marker: Final = unique_marker() model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response = client.proxy.transport.send( + response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), json=ChatBody( model=model_name, - messages=[ChatMessage(role="user", content=f"{marker} scripted anthropic stream")], + messages=(ChatMessage(role="user", content=f"{marker} scripted anthropic stream"),), stream=True, stream_options=ChatStreamOptions(include_usage=True), ), @@ -172,8 +177,8 @@ class TestWireFormats: assert response.stream_done, "anthropic stream did not reach its terminal event" assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - expected = expected_breakdown(model, case) - row = cost_rows.poll_cost_row_where( + expected: Final = expected_breakdown(model, case) + row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, From 99bf8e9b2ffb6c647813029debba23c788e86b41 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 15 Sep 2026 23:32:39 +0000 Subject: [PATCH 03/30] test(e2e): add cost calculation CI proxy config Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/gateway/cost_calculation_ci_config.yml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/gateway/cost_calculation_ci_config.yml diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index b6c3840f626..49cfc29aa17 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/gateway/cost_calculation_ci_config.yml b/tests/e2e/gateway/cost_calculation_ci_config.yml new file mode 100644 index 00000000000..ac0603fa7c1 --- /dev/null +++ b/tests/e2e/gateway/cost_calculation_ci_config.yml @@ -0,0 +1,7 @@ +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + proxy_batch_write_at: 5 + +model_list: [] From 415b06f5ff6d5959e2144ea826922694b2c8a60b Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 04:42:07 +0000 Subject: [PATCH 04/30] test(e2e): assert the real bill for the four fixed cost gaps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cost_matrix.py | 22 +++++-------------- .../test_token_pricing_e2e.py | 5 ----- tests/e2e/cost_map.json | 15 +++++++++++++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index e8b1d249559..8f39e89a358 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -186,15 +186,12 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ } ), "openai_responses": frozenset({"cache_read", "reasoning", "web_search", "response_model", "absent_usage"}), - # Product gap: litellm hard-indexes message_delta["usage"] in - # anthropic/chat/handler.py, so a usage-absent anthropic stream raises - # KeyError; the real wire always carries it, so the case cannot be - # represented. - "anthropic_messages": frozenset({"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model"}), - # Product gap: the gemini transform sets ModelResponse.model from the - # request and drops the provider's modelVersion, so a response-model - # override can never be priced on this wire. - "gemini_generate": frozenset({"cache_read", "reasoning", "audio", "web_search", "absent_usage"}), + "anthropic_messages": frozenset( + {"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model", "absent_usage"} + ), + "gemini_generate": frozenset( + {"cache_read", "reasoning", "audio", "web_search", "response_model", "absent_usage"} + ), "together_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", @@ -240,9 +237,6 @@ class Case: billed_web_search_calls: int = 0 response_model_override: bool = False exact_spend: bool = True - # stream_usage=absent on a wire with no proxy-side token recount means the - # bill is exactly zero; asserted as such rather than skipped. - expect_zero_bill: bool = False def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( @@ -359,10 +353,6 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: stream=True, stream_usage="absent", exact_spend=False, - # The responses surface bills only provider-reported usage; - # with no usage in the stream the spend row is zero. Other - # wires recount tokens proxy-side and bill a nonzero amount. - expect_zero_bill=model.wire == "openai_responses", ) if "absent_usage" in caps else None diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index e210dad94b1..ead86931424 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -90,11 +90,6 @@ class TestTokenPricing: ) assert row is not None, f"no spend row with a cost breakdown landed for {model.map_key}/{case.name}" - if not case.exact_spend and case.expect_zero_bill: - # The provider reported no usage and this wire has no proxy-side - # recount, so the bill is exactly zero. - assert row.spend is not None and row.spend == 0, f"no-usage stream billed {row.spend}: {row}" - return if not case.exact_spend: # stream_usage=absent: the provider reported no usage, so the row's # token counts are the proxy's own recount; only assert a bill landed. diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index b761710bae3..68d840870c9 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -63,13 +63,18 @@ "supports_web_search": true }, "fireworks_ai/deepseek-v4p1-flash": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, "cache_read_input_token_cost": 1.4e-05, + "input_cost_per_audio_token": 0.00066, "input_cost_per_token": 0.00014000000000000001, "litellm_provider": "fireworks_ai", "max_input_tokens": 2000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, "output_cost_per_token": 0.00028000000000000003, "search_context_cost_per_query": { "search_context_size_high": 0.03, @@ -82,13 +87,18 @@ "supports_web_search": true }, "fireworks_ai/kimi-k3": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, "cache_read_input_token_cost": 1.2e-05, + "input_cost_per_audio_token": 0.00066, "input_cost_per_token": 0.00012000000000000002, "litellm_provider": "fireworks_ai", "max_input_tokens": 2000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, "output_cost_per_token": 0.00024000000000000003, "search_context_cost_per_query": { "search_context_size_high": 0.03, @@ -101,13 +111,18 @@ "supports_web_search": true }, "fireworks_ai/qwen3p8-max": { + "cache_creation_input_token_cost": 0.00033, + "cache_creation_input_token_cost_above_1hr": 0.00044, "cache_read_input_token_cost": 1.3e-05, + "input_cost_per_audio_token": 0.00066, "input_cost_per_token": 0.00013000000000000002, "litellm_provider": "fireworks_ai", "max_input_tokens": 2000000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", + "output_cost_per_audio_token": 0.00077, + "output_cost_per_reasoning_token": 0.00055, "output_cost_per_token": 0.00026000000000000003, "search_context_cost_per_query": { "search_context_size_high": 0.03, From feb69c5f789d44a65dbbfa348ce39eaa3874b37f Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 16:38:15 +0000 Subject: [PATCH 05/30] test(e2e): add tool-call, terminal, and image-input shapes to the cost matrix Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cost_matrix.py | 184 +++++++- .../e2e/cost_calculation/scripted_provider.py | 408 +++++++++++++++--- .../test_token_pricing_e2e.py | 61 ++- .../cost_calculation/test_wire_formats_e2e.py | 111 ++++- 4 files changed, 688 insertions(+), 76 deletions(-) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 8f39e89a358..5f634712778 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -17,7 +17,11 @@ creation), the case is absent from the matrix rather than silently zero. from __future__ import annotations +import base64 import json +import random +import struct +import zlib from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -26,7 +30,7 @@ from typing import Final, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, TypeAdapter -from scripted_provider import Scenario, ScriptedOutput, ScriptedUsage, Wire +from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" @@ -182,26 +186,37 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ "openai_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", + } + ), + "openai_responses": frozenset( + { + "cache_read", "reasoning", "web_search", "response_model", "absent_usage", + "tool_call", "image_input", "responses_terminal", } ), - "openai_responses": frozenset({"cache_read", "reasoning", "web_search", "response_model", "absent_usage"}), "anthropic_messages": frozenset( - {"cache_read", "cache_write_5m", "cache_write_1h", "web_search", "response_model", "absent_usage"} + { + "cache_read", "cache_write_5m", "cache_write_1h", "web_search", + "response_model", "absent_usage", "tool_call", "image_input", + } ), "gemini_generate": frozenset( - {"cache_read", "reasoning", "audio", "web_search", "response_model", "absent_usage"} + { + "cache_read", "reasoning", "audio", "web_search", "response_model", + "absent_usage", "tool_call", "image_input", "prompt_blocked", + } ), "together_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", } ), "fireworks_chat": frozenset( { "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", } ), }) @@ -220,6 +235,15 @@ CaseName: TypeAlias = Literal[ "stream", "stream_no_usage", "response_model_override", + "stream_response_model_override", + "tool_call", + "stream_no_usage_tool_call", + "stream_no_usage_image_input", + "stream_no_usage_incomplete", + "stream_unvalidated", + "stream_no_usage_unvalidated", + "prompt_blocked", + "stream_prompt_blocked", ] @@ -237,6 +261,9 @@ class Case: billed_web_search_calls: int = 0 response_model_override: bool = False exact_spend: bool = True + tool_call: bool = False + image_input: bool = False + terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( @@ -246,6 +273,10 @@ class Case: output=ScriptedOutput( text=text, response_model=model.override_model if self.response_model_override else None, + tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) + if self.tool_call + else None, + terminal=self.terminal, ), stream_usage=self.stream_usage, service_tier=self.service_tier, @@ -254,6 +285,15 @@ class Case: _BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) +TOOL_CALL_ARGUMENTS: Final = json.dumps({ + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler " * 30, +}) + +_PROMPT_BLOCKED_USAGE: Final = ScriptedUsage(fresh_input_tokens=1000, output_tokens=0) + def _web_search_case(model: FrontierModel) -> Case: counts_exactly: Final = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") @@ -362,6 +402,100 @@ def cases_for(model: FrontierModel) -> tuple[Case, ...]: if "response_model" in caps else None ), + ( + Case( + name="stream_response_model_override", + usage=_BASIC_USAGE, + stream=True, + response_model_override=True, + ) + if "response_model" in caps + else None + ), + ( + Case(name="tool_call", usage=_BASIC_USAGE, tool_call=True) + if "tool_call" in caps + else None + ), + ( + Case( + name="stream_no_usage_tool_call", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + tool_call=True, + exact_spend=False, + ) + if "absent_usage" in caps and "tool_call" in caps + else None + ), + ( + Case( + name="stream_no_usage_image_input", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + image_input=True, + exact_spend=False, + ) + if "absent_usage" in caps and "image_input" in caps + else None + ), + ( + Case( + name="stream_no_usage_incomplete", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + terminal="incomplete", + exact_spend=False, + ) + if "responses_terminal" in caps + else None + ), + ( + Case( + name="stream_unvalidated", + usage=_BASIC_USAGE, + stream=True, + terminal="unvalidated", + ) + if "responses_terminal" in caps + else None + ), + ( + Case( + name="stream_no_usage_unvalidated", + usage=_BASIC_USAGE, + stream=True, + stream_usage="absent", + terminal="unvalidated", + exact_spend=False, + ) + if "responses_terminal" in caps + else None + ), + ( + Case( + name="prompt_blocked", + usage=_PROMPT_BLOCKED_USAGE, + terminal="prompt_blocked", + response_model_override=True, + ) + if "prompt_blocked" in caps + else None + ), + ( + Case( + name="stream_prompt_blocked", + usage=_PROMPT_BLOCKED_USAGE, + stream=True, + terminal="prompt_blocked", + response_model_override=True, + ) + if "prompt_blocked" in caps + else None + ), ) return tuple(case for case in candidates if case is not None) @@ -436,6 +570,42 @@ def expected_cost(model: FrontierModel, case: Case) -> float: return expected_breakdown(model, case).total +def recount_cost( + model: FrontierModel, case: Case, prompt_tokens: int, completion_tokens: int +) -> float: + """What the proxy's own token recount should cost at the case's rates, + without pinning the tokenizer's exact counts.""" + rates: Final = model.override_rates if case.response_model_override else model.rates + return prompt_tokens * (rates.input_cost_per_token or 0.0) + completion_tokens * ( + rates.output_cost_per_token or 0.0 + ) + + +def _png_chunk(tag: bytes, payload: bytes) -> bytes: + return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", zlib.crc32(tag + payload)) + + +def image_input_data_url() -> str: + """A deterministic 256x256 RGB noise PNG as a data URL; noise compresses + poorly on purpose so the base64 payload stays well above 100 KB and would + blow up the prompt recount if the URL were ever tokenized as text.""" + rng: Final = random.Random(0) + side: Final = 256 + raw: Final = b"".join( + b"\x00" + rng.randbytes(side * 3) for _ in range(side) + ) + png: Final = ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)) + + _png_chunk(b"IDAT", zlib.compress(raw)) + + _png_chunk(b"IEND", b"") + ) + return "data:image/png;base64," + base64.b64encode(png).decode() + + +IMAGE_INPUT_DATA_URL: Final = image_input_data_url() + + def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: """(prompt_tokens, completion_tokens) the spend row should carry, per the wire's normalization: Anthropic folds cache read/write into prompt_tokens, diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index f1deafd1bc5..e1a6c430307 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -38,7 +38,7 @@ from types import MappingProxyType from typing import Final, Literal, TypeAlias from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, model_validator Wire: TypeAlias = Literal[ "openai_chat", @@ -62,6 +62,26 @@ WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( StreamUsage: TypeAlias = Literal["final_chunk", "absent"] ServiceTier: TypeAlias = Literal["flex", "priority"] +TerminalKind: TypeAlias = Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] + +# Which terminal variant each wire can represent. +_TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + "openai_responses": frozenset({"incomplete", "unvalidated"}), + "gemini_generate": frozenset({"prompt_blocked"}), + } +) + + +class ScriptedToolCall(BaseModel): + """A single function call the scripted output emits instead of text. + ``arguments`` is the wire's JSON string (~250 chars), sliced into deltas + for streams.""" + + model_config = ConfigDict(frozen=True) + + name: str + arguments: str class ScriptedUsage(BaseModel): @@ -96,6 +116,12 @@ class ScriptedOutput(BaseModel): # OpenAI-compatible providers can report a provider-computed cost; emitted as # the top-level "cost" field on the together/fireworks wire. provider_cost: float | None = None + # When set, the response is a tool call only: no text content on any wire. + tool_call: ScriptedToolCall | None = None + # Terminal shape: "unvalidated" makes the Responses terminal response fail + # pydantic validation so the proxy takes its model_construct dict path; + # "prompt_blocked" is a Gemini promptFeedback-only body. + terminal: TerminalKind = "completed" class Scenario(BaseModel): @@ -108,6 +134,17 @@ class Scenario(BaseModel): stream_usage: StreamUsage = "final_chunk" service_tier: ServiceTier | None = None + @model_validator(mode="after") + def _check_terminal_supported(self) -> Scenario: + if ( + self.output.terminal != "completed" + and self.output.terminal not in _TERMINAL_CAPS.get(self.wire, frozenset()) + ): + raise ValueError( + f"wire {self.wire} cannot emit terminal={self.output.terminal}" + ) + return self + @property def mount(self) -> str: return WIRE_MOUNTS[self.wire] @@ -291,10 +328,38 @@ def _responses_usage(u: ScriptedUsage) -> Mapping[str, object]: # ---------- per-wire responses ---------- +def _split_arguments(arguments: str) -> tuple[str, ...]: + """Slice a tool-call arguments JSON string into 2-3 streamed deltas.""" + third: Final = max(1, len(arguments) // 3) + return tuple( + slice_ + for slice_ in (arguments[:third], arguments[third : 2 * third], arguments[2 * third :]) + if slice_ + ) + + def _openai_message(scenario: Scenario) -> Mapping[str, object]: + tool_call: Final = scenario.output.tool_call return _jobj_opt( ("role", "assistant"), - ("content", scenario.output.text), + ("content", None if tool_call is not None else scenario.output.text), + ( + ( + "tool_calls", + ( + _jobj( + ("id", f"call_{scenario.scenario_id}"), + ("type", "function"), + ( + "function", + _jobj(("name", tool_call.name), ("arguments", tool_call.arguments)), + ), + ), + ), + ) + if tool_call is not None + else None + ), ( ( "annotations", @@ -332,7 +397,12 @@ def _openai_chat_body(scenario: Scenario, requested_model: str) -> Mapping[str, _jobj( ("index", 0), ("message", _openai_message(scenario)), - ("finish_reason", scenario.output.finish_reason), + ( + "finish_reason", + "tool_calls" + if scenario.output.tool_call is not None + else scenario.output.finish_reason, + ), ), ), ), @@ -359,6 +429,7 @@ def _openai_chunk( def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: + tool_call: Final = scenario.output.tool_call delta: Final = _jobj_opt( ("role", "assistant"), ("content", scenario.output.text), @@ -368,6 +439,43 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: else None ), ) + body_deltas: Final[tuple[Mapping[str, object], ...]] = ( + ( + _jobj( + ("role", "assistant"), + ( + "tool_calls", + ( + _jobj( + ("index", 0), + ("id", f"call_{scenario.scenario_id}"), + ("type", "function"), + ( + "function", + _jobj(("name", tool_call.name), ("arguments", "")), + ), + ), + ), + ), + ), + *( + _jobj( + ( + "tool_calls", + ( + _jobj( + ("index", 0), + ("function", _jobj(("arguments", arguments_slice))), + ), + ), + ) + ) + for arguments_slice in _split_arguments(tool_call.arguments) + ), + ) + if tool_call is not None + else (delta,) + ) return _sse( ( ( @@ -378,13 +486,16 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: choices=(_jobj(("index", 0), ("delta", _jobj(("role", "assistant"))), ("finish_reason", None)),), ), ), - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=(_jobj(("index", 0), ("delta", delta), ("finish_reason", None)),), - ), + *( + ( + None, + _openai_chunk( + scenario, + requested_model, + choices=(_jobj(("index", 0), ("delta", body_delta), ("finish_reason", None)),), + ), + ) + for body_delta in body_deltas ), ( None, @@ -395,7 +506,12 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: _jobj( ("index", 0), ("delta", _jobj()), - ("finish_reason", scenario.output.finish_reason), + ( + "finish_reason", + "tool_calls" + if tool_call is not None + else scenario.output.finish_reason, + ), ), ), ), @@ -410,17 +526,34 @@ def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: ) +def _anthropic_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + if tool_call is not None: + return ( + _jobj( + ("type", "tool_use"), + ("id", f"toolu_{scenario.scenario_id}"), + ("name", tool_call.name), + ("input", json.loads(tool_call.arguments)), + ), + ) + return (_jobj(("type", "text"), ("text", scenario.output.text)),) + + +def _anthropic_stop_reason(scenario: Scenario) -> str: + if scenario.output.tool_call is not None: + return "tool_use" + return "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason + + def _anthropic_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: return _jobj( ("id", f"msg_{scenario.scenario_id}"), ("type", "message"), ("role", "assistant"), ("model", scenario.output.response_model or requested_model), - ("content", (_jobj(("type", "text"), ("text", scenario.output.text)),)), - ( - "stop_reason", - "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, - ), + ("content", _anthropic_content(scenario)), + ("stop_reason", _anthropic_stop_reason(scenario)), ("usage", _anthropic_usage(scenario.usage)), ) @@ -453,12 +586,7 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: ("type", "message_delta"), ( "delta", - _jobj( - ( - "stop_reason", - "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason, - ) - ), + _jobj(("stop_reason", _anthropic_stop_reason(scenario))), ), ( ("usage", _jobj(("output_tokens", scenario.usage.output_tokens))) @@ -474,16 +602,45 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: _jobj( ("type", "content_block_start"), ("index", 0), - ("content_block", _jobj(("type", "text"), ("text", ""))), + ( + "content_block", + _jobj( + ("type", "tool_use"), + ("id", f"toolu_{scenario.scenario_id}"), + ("name", scenario.output.tool_call.name), + ("input", _jobj()), + ) + if scenario.output.tool_call is not None + else _jobj(("type", "text"), ("text", "")), + ), ), ), - ( - "content_block_delta", - _jobj( - ("type", "content_block_delta"), - ("index", 0), - ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), - ), + *( + tuple( + ( + "content_block_delta", + _jobj( + ("type", "content_block_delta"), + ("index", 0), + ( + "delta", + _jobj(("type", "input_json_delta"), ("partial_json", arguments_slice)), + ), + ), + ) + for arguments_slice in _split_arguments(scenario.output.tool_call.arguments) + ) + if scenario.output.tool_call is not None + else ( + ( + "content_block_delta", + _jobj( + ("type", "content_block_delta"), + ("index", 0), + ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), + ), + ), + ) ), ("content_block_stop", _jobj(("type", "content_block_stop"), ("index", 0))), ("message_delta", message_delta), @@ -492,7 +649,49 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: ) +def _gemini_prompt_blocked_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + return _jobj( + ( + "promptFeedback", + _jobj( + ("blockReason", "SAFETY"), + ( + "safetyRatings", + ( + _jobj( + ("category", "HARM_CATEGORY_HARASSMENT"), + ("probability", "HIGH"), + ("blocked", True), + ), + ), + ), + ), + ), + ("usageMetadata", _gemini_usage(scenario.usage)), + ("modelVersion", scenario.output.response_model or requested_model), + ) + + +def _gemini_parts(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + if tool_call is not None: + return ( + _jobj( + ( + "functionCall", + _jobj( + ("name", tool_call.name), + ("args", json.loads(tool_call.arguments)), + ), + ) + ), + ) + return (_jobj(("text", scenario.output.text)),) + + def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + if scenario.output.terminal == "prompt_blocked": + return _gemini_prompt_blocked_body(scenario, requested_model) return _jobj( ( "candidates", @@ -501,7 +700,7 @@ def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, objec ( "content", _jobj( - ("parts", (_jobj(("text", scenario.output.text)),)), + ("parts", _gemini_parts(scenario)), ("role", "model"), ), ), @@ -559,67 +758,148 @@ def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: ) -def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - return _jobj( - ("id", f"resp_{scenario.scenario_id}"), - ("object", "response"), - ("created_at", int(time.time())), - ("status", "completed"), - ("model", scenario.output.response_model or requested_model), - ( - "output", +def _responses_output(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + return ( + *( ( - *( - _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) - for i in range(scenario.usage.web_search_calls) - ), - _jobj( - ("type", "message"), - ("id", f"msg_{scenario.scenario_id}"), - ("status", "completed"), - ("role", "assistant"), - ( - "content", - ( - _jobj( - ("type", "output_text"), - ("text", scenario.output.text), - ("annotations", ()), - ), - ), + _jobj(("type", "scripted_future_item"), ("id", f"fut_{scenario.scenario_id}"), ("status", "completed")), + ) + if scenario.output.terminal == "unvalidated" + else () + ), + *( + _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) + for i in range(scenario.usage.web_search_calls) + ), + _jobj( + ("type", "function_call"), + ("id", f"fc_{scenario.scenario_id}"), + ("call_id", f"call_{scenario.scenario_id}"), + ("name", tool_call.name), + ("arguments", tool_call.arguments), + ("status", "completed"), + ) + if tool_call is not None + else _jobj( + ("type", "message"), + ("id", f"msg_{scenario.scenario_id}"), + ("status", "completed"), + ("role", "assistant"), + ( + "content", + ( + _jobj( + ("type", "output_text"), + ("text", scenario.output.text), + ("annotations", ()), ), ), ), ), + ) + + +def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: + incomplete: Final = scenario.output.terminal == "incomplete" + return _jobj_opt( + ("id", f"resp_{scenario.scenario_id}"), + ("object", "response"), + ( + "created_at", + "not-a-number" if scenario.output.terminal == "unvalidated" else int(time.time()), + ), + ("status", "incomplete" if incomplete else "completed"), + ( + ("incomplete_details", _jobj(("reason", "max_output_tokens"))) + if incomplete + else None + ), + ("model", scenario.output.response_model or requested_model), + ("output", _responses_output(scenario)), ("usage", _responses_usage(scenario.usage)), ) def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: - completed: Final = ( + tool_call: Final = scenario.output.tool_call + terminal: Final = ( _jobj(*((key, value) for key, value in _responses_body(scenario, requested_model).items() if key != "usage")) if scenario.stream_usage == "absent" else _responses_body(scenario, requested_model) ) created: Final = _jobj( - *((key, value) for key, value in completed.items() if key not in ("status", "usage")), + *((key, value) for key, value in terminal.items() if key not in ("status", "usage")), ("status", "in_progress"), ("usage", None), ) - return _sse( + terminal_event: Final = ( + "response.incomplete" if scenario.output.terminal == "incomplete" else "response.completed" + ) + output_index: Final = ( + scenario.usage.web_search_calls + (1 if scenario.output.terminal == "unvalidated" else 0) + ) + middle_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( ( - ("response.created", _jobj(("type", "response.created"), ("response", created))), + ( + "response.output_item.added", + _jobj( + ("type", "response.output_item.added"), + ("output_index", output_index), + ( + "item", + _jobj( + ("type", "function_call"), + ("id", f"fc_{scenario.scenario_id}"), + ("call_id", f"call_{scenario.scenario_id}"), + ("name", tool_call.name), + ("arguments", ""), + ("status", "in_progress"), + ), + ), + ), + ), + *( + ( + "response.function_call_arguments.delta", + _jobj( + ("type", "response.function_call_arguments.delta"), + ("item_id", f"fc_{scenario.scenario_id}"), + ("output_index", output_index), + ("delta", arguments_slice), + ), + ) + for arguments_slice in _split_arguments(tool_call.arguments) + ), + ( + "response.function_call_arguments.done", + _jobj( + ("type", "response.function_call_arguments.done"), + ("item_id", f"fc_{scenario.scenario_id}"), + ("output_index", output_index), + ("arguments", tool_call.arguments), + ), + ), + ) + if tool_call is not None + else ( ( "response.output_text.delta", _jobj( ("type", "response.output_text.delta"), ("item_id", f"msg_{scenario.scenario_id}"), - ("output_index", scenario.usage.web_search_calls), + ("output_index", output_index), ("content_index", 0), ("delta", scenario.output.text), ), ), - ("response.completed", _jobj(("type", "response.completed"), ("response", completed))), + ) + ) + return _sse( + ( + ("response.created", _jobj(("type", "response.created"), ("response", created))), + *middle_events, + (terminal_event, _jobj(("type", terminal_event), ("response", terminal))), ) ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index ead86931424..0b4f3e1fd37 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -16,15 +16,26 @@ from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( FRONTIER_MODELS, + IMAGE_INPUT_DATA_URL, Case, FrontierModel, cases_for, expected_cost, expected_token_columns, + recount_cost, ) from e2e_config import unique_marker from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamOptions +from models import ( + ChatBody, + ChatMessage, + ChatStreamOptions, + ChatTool, + ChatToolFunction, + ImageContentPart, + ImageUrl, + TextContentPart, +) pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark @@ -41,10 +52,37 @@ def _case_id(param: tuple[FrontierModel, Case]) -> str: def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: return ChatBody( model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted pricing call"),), + messages=( + ChatMessage( + role="user", + content=( + [ + TextContentPart(text=f"{marker} scripted pricing call"), + ImageContentPart(image_url=ImageUrl(url=IMAGE_INPUT_DATA_URL)), + ] + if case.image_input + else f"{marker} scripted pricing call" + ), + ), + ), stream=case.stream, stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, service_tier=case.service_tier, + tools=( + ( + ChatTool( + function=ChatToolFunction( + name="get_weather", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + ) + ), + ) + if case.tool_call + else None + ), ) @@ -92,8 +130,23 @@ class TestTokenPricing: if not case.exact_spend: # stream_usage=absent: the provider reported no usage, so the row's - # token counts are the proxy's own recount; only assert a bill landed. - assert row.spend is not None and row.spend > 0, f"no-usage stream billed nothing: {row}" + # token counts are the proxy's own recount; assert the recount + # billed both directions at the case's rates. + assert row.prompt_tokens is not None and row.prompt_tokens > 0, ( + f"no-usage stream counted no input tokens: {row}" + ) + assert row.completion_tokens is not None and row.completion_tokens > 0, ( + f"no-usage stream counted no output tokens: {row}" + ) + if case.image_input: + assert row.prompt_tokens < 4000, ( + f"image data URL looks tokenized as text: prompt_tokens={row.prompt_tokens}" + ) + assert row.spend is not None and cost_rows.approx_equal( + row.spend, + recount_cost(model, case, row.prompt_tokens, row.completion_tokens), + ), f"no-usage stream spend {row.spend} != recount at map rates: {row}" + cost_rows.assert_total_is_sum_of_components(row) return assert row.spend is not None and cost_rows.approx_equal(row.spend, expected), ( diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index c0276cf370c..3c6c34b24fb 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -26,7 +26,7 @@ from cost_matrix import ( ) from e2e_config import unique_marker from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamOptions +from models import ChatBody, ChatMessage, ChatStreamOptions, ChatTool, ChatToolFunction from scripted_provider import ScriptedUsage pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark @@ -96,6 +96,53 @@ _WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ ), }) +_SHAPE_USAGE: Final = ScriptedUsage(fresh_input_tokens=80, output_tokens=25) + +# Renderer-level shapes the pricing matrix gates per cap, pinned here once per +# wire so the sidecar emits prove they survive the proxy end to end. +_SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( + *( + ( + f"tool_call_{'stream' if stream else 'sync'}", + wire, + Case(name="tool_call", usage=_SHAPE_USAGE, stream=stream, tool_call=True), + ) + for wire in _WIRE_USAGE + for stream in (False, True) + ), + ( + "responses_incomplete", + "openai_responses", + Case(name="stream_no_usage_incomplete", usage=_SHAPE_USAGE, stream=True, terminal="incomplete"), + ), + ( + "responses_unvalidated", + "openai_responses", + Case(name="stream_unvalidated", usage=_SHAPE_USAGE, stream=True, terminal="unvalidated"), + ), + ( + "gemini_prompt_blocked", + "gemini_generate", + Case( + name="prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + terminal="prompt_blocked", + response_model_override=True, + ), + ), + ( + "gemini_prompt_blocked_stream", + "gemini_generate", + Case( + name="stream_prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + stream=True, + terminal="prompt_blocked", + response_model_override=True, + ), + ), +) + class TestWireFormats: @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) @@ -189,3 +236,65 @@ class TestWireFormats: f"(breakdown {row.breakdown.model_dump()})" ) cost_rows.assert_total_is_sum_of_components(row) + + @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=lambda entry: entry[0]) + @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") + def test_response_shape_bills_reported_usage( + self, + client: CostCalcClient, + resources: ResourceManager, + scoped_key: str, + shape_wire_case: tuple[str, str, Case], + ) -> None: + shape, wire, case = shape_wire_case + map_key, _usage = _WIRE_USAGE[wire] + model: Final = _MODELS[map_key] + marker: Final = unique_marker() + model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) + response: Final = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model_name, + messages=(ChatMessage(role="user", content=f"{marker} scripted {shape}"),), + stream=case.stream, + stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, + tools=( + ( + ChatTool( + function=ChatToolFunction( + name="get_weather", + parameters={"type": "object", "properties": {"city": {"type": "string"}}}, + ) + ), + ) + if case.tool_call + else None + ), + ), + stream=case.stream, + ) + assert response.ok, f"{shape}: proxy returned {response.status_code}: {response.body[:400]}" + if case.stream: + assert response.stream_done, f"{shape}: stream did not reach its terminal event" + assert response.stream_error is None, f"{shape}: stream error: {response.stream_error}" + + expected: Final = expected_breakdown(model, case) + row: Final = cost_rows.poll_cost_row_where( + client.proxy, + scoped_key, + lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, + ) + assert row is not None, f"{shape}: no spend row landed" + assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( + f"{shape}: spend {row.spend} != expected {expected.total} " + f"(breakdown {row.breakdown.model_dump()})" + ) + prompt_tokens, completion_tokens = expected_token_columns(model, case) + assert row.prompt_tokens == prompt_tokens, ( + f"{shape}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + ) + assert row.completion_tokens == completion_tokens, ( + f"{shape}: completion_tokens {row.completion_tokens} != {completion_tokens}" + ) + cost_rows.assert_total_is_sum_of_components(row) From 5507de326e3e98f9069af5f9d1315c89bb3c3e25 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 16:45:09 +0000 Subject: [PATCH 06/30] test(e2e): type the wire-shape parametrize ids callback Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/test_wire_formats_e2e.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index 3c6c34b24fb..4da7b31a6ef 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -144,6 +144,10 @@ _SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( ) +def _shape_id(entry: tuple[str, str, Case]) -> str: + return entry[0] + + class TestWireFormats: @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") @@ -237,7 +241,7 @@ class TestWireFormats: ) cost_rows.assert_total_is_sum_of_components(row) - @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=lambda entry: entry[0]) + @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=_shape_id) @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") def test_response_shape_bills_reported_usage( self, From 9885dc89621697e235fc65e0e86e147da87398c1 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 22:52:42 +0000 Subject: [PATCH 07/30] test(e2e): add azure, bedrock converse and vertex wires to the cost suite Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/conftest.py | 54 +++- tests/e2e/cost_calculation/cost_matrix.py | 144 ++++++++- .../e2e/cost_calculation/scripted_provider.py | 284 +++++++++++++++++- .../cost_calculation/test_wire_formats_e2e.py | 64 ++++ tests/e2e/cost_map.json | 158 ++++++++++ tests/e2e/models.py | 1 + 6 files changed, 681 insertions(+), 24 deletions(-) diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 345ca26f7e3..8c6db7c0010 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -12,6 +12,7 @@ Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). from __future__ import annotations import importlib.util +import json import sys from collections.abc import Callable, Mapping from dataclasses import dataclass @@ -22,7 +23,7 @@ from typing import Final, Protocol, cast import pytest from cost_matrix import Case, FrontierModel -from e2e_config import COST_MAP_PROXY_URL +from e2e_config import COST_MAP_PROXY_URL, SCRIPTED_PROVIDER_PROXY_BASE from lifecycle import ResourceManager from models import LiteLLMParamsBody, ModelInfoBody, ModelNewBody from proxy_client import ProxyClient, build_proxy_client @@ -111,6 +112,41 @@ def client() -> CostCalcClient: return CostCalcClient(proxy=proxy) +_vertex_key_pem: str | None = None + + +def _vertex_service_account_json() -> str: + """A service-account credential JSON whose token_uri is the sidecar's + /_oauth/token route: the proxy's google-auth refresh then gets a scripted + access token without touching Google. One generated RSA key per process.""" + global _vertex_key_pem # mutable-ok: session-scoped key generation cached for reuse + if _vertex_key_pem is None: + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric import rsa + + _vertex_key_pem = ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) + .private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + .decode() + ) + return json.dumps( + { + "type": "service_account", + "project_id": "cc-scripted-project", + "private_key_id": "scripted", + "private_key": _vertex_key_pem, + "client_email": "scripted@cc-scripted-project.iam.gserviceaccount.com", + "client_id": "0", + "auth_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/authorize", + "token_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/token", + } + ) + + def register_scenario_deployment( client: CostCalcClient, resources: ResourceManager, @@ -126,15 +162,21 @@ def register_scenario_deployment( handle: Final = register_scenario(scenario) resources.defer(lambda: delete_scenario(handle)) model_name: Final = f"{model.model_name}-{marker}" + extra_params: Final[dict[str, str]] = dict(model.litellm_params) + if model.wire == "vertex_generate": + extra_params["vertex_credentials"] = _vertex_service_account_json() model_id: Final = client.proxy.register_model( ModelNewBody( model_name=model_name, - litellm_params=LiteLLMParamsBody( - model=model.litellm_model, - api_key=model.api_key, - api_base=handle.api_base(), + litellm_params=LiteLLMParamsBody.model_validate( + { + "model": model.litellm_model, + "api_key": model.api_key, + "api_base": handle.api_base(), + **extra_params, + } ), - model_info=ModelInfoBody(), + model_info=ModelInfoBody(base_model=model.base_model), ) ) resources.defer(lambda: client.proxy.delete_model(model_id)) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 5f634712778..37495f37b0b 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -88,7 +88,14 @@ class FrontierModel: litellm_model: str wire: Wire map_key: str - override_model: str + override_model: str | None = None + override_map_key: str | None = None + # Registered as model_info.base_model; when set, the provider-reported + # model loses to it and every case bills at this deployment's own rates. + base_model: str | None = None + # Extra litellm_params merged into the /model/new registration (api_version, + # aws_* credentials, vertex_* auth). + litellm_params: Mapping[str, str] = MappingProxyType({}) @property def rates(self) -> CostMapEntry: @@ -96,11 +103,16 @@ class FrontierModel: @property def override_rates(self) -> CostMapEntry: + if self.base_model is not None or self.override_map_key is None: + return self.rates return _COST_MAP[self.override_map_key] @property - def override_map_key(self) -> str: - return _OVERRIDE_MAP_KEYS[self.override_model] + def provider_model(self) -> str: + """The bare provider-facing model name: litellm_model minus the provider + prefix and any routing segment (converse/, responses/).""" + tail: Final = self.litellm_model.split("/")[1:] + return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) @property def provider(self) -> str: @@ -166,6 +178,92 @@ _FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( ) +@dataclass(frozen=True, slots=True) +class _ExtendedSpec: + """A frontier entry whose override target, model_info.base_model or extra + litellm_params can't be derived from the map key alone.""" + + map_key: str + litellm_model: str + wire: Wire + override_model: str | None = None + override_map_key: str | None = None + base_model: str | None = None + litellm_params: Mapping[str, str] = MappingProxyType({}) + + +_AZURE_PARAMS: Final[Mapping[str, str]] = MappingProxyType({"api_version": "2025-04-01-preview"}) +_BEDROCK_PARAMS: Final[Mapping[str, str]] = MappingProxyType( + { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + } +) +_VERTEX_PARAMS: Final[Mapping[str, str]] = MappingProxyType( + { + "vertex_project": "cc-scripted-project", + "vertex_location": "us-central1", + } +) + +_EXTENDED_SPECS: Final[tuple[_ExtendedSpec, ...]] = ( + _ExtendedSpec( + map_key="azure/gpt-5.6", + litellm_model="azure/gpt-5.6", + wire="azure_chat", + override_model="gpt-5.4-mini", + override_map_key="azure/gpt-5.4-mini", + litellm_params=_AZURE_PARAMS, + ), + _ExtendedSpec( + # Deployment name is not a model; base_model pins billing so the + # response's model field loses, proving base_model wins. + map_key="azure/gpt-5.4-mini", + litellm_model="azure/cc-pinned-deployment", + wire="azure_chat", + override_model="gpt-5.6", + override_map_key="azure/gpt-5.6", + base_model="azure/gpt-5.4-mini", + litellm_params=_AZURE_PARAMS, + ), + _ExtendedSpec( + map_key="anthropic.claude-sonnet-5-v1:0", + litellm_model="bedrock/converse/anthropic.claude-sonnet-5-v1:0", + wire="bedrock_converse", + litellm_params=_BEDROCK_PARAMS, + ), + _ExtendedSpec( + map_key="us.anthropic.claude-opus-5-v1:0", + litellm_model="bedrock/converse/us.anthropic.claude-opus-5-v1:0", + wire="bedrock_converse", + litellm_params=_BEDROCK_PARAMS, + ), + _ExtendedSpec( + map_key="meta.llama4-maverick-17b-instruct-v1:0", + litellm_model="bedrock/converse/meta.llama4-maverick-17b-instruct-v1:0", + wire="bedrock_converse", + litellm_params=_BEDROCK_PARAMS, + ), + _ExtendedSpec( + map_key="gemini-3.8-flash", + litellm_model="vertex_ai/gemini-3.8-flash", + wire="vertex_generate", + override_model="gemini-3.1-pro-preview", + override_map_key="gemini-3.1-pro-preview", + litellm_params=_VERTEX_PARAMS, + ), + _ExtendedSpec( + map_key="gemini-3.1-pro-preview", + litellm_model="vertex_ai/gemini-3.1-pro-preview", + wire="vertex_generate", + override_model="gemini-3.8-flash", + override_map_key="gemini-3.8-flash", + litellm_params=_VERTEX_PARAMS, + ), +) + + def _frontier() -> tuple[FrontierModel, ...]: return tuple( FrontierModel( @@ -174,8 +272,21 @@ def _frontier() -> tuple[FrontierModel, ...]: wire=wire, map_key=map_key, override_model=_OVERRIDE_MODELS[map_key], + override_map_key=_OVERRIDE_MAP_KEYS[_OVERRIDE_MODELS[map_key]], ) for map_key, litellm_model, wire in _FRONTIER_SPECS + ) + tuple( + FrontierModel( + model_name=f"cc-{spec.map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", + litellm_model=spec.litellm_model, + wire=spec.wire, + map_key=spec.map_key, + override_model=spec.override_model, + override_map_key=spec.override_map_key, + base_model=spec.base_model, + litellm_params=spec.litellm_params, + ) + for spec in _EXTENDED_SPECS ) @@ -219,6 +330,24 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ "web_search", "response_model", "absent_usage", "tool_call", "image_input", } ), + "azure_chat": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", + "web_search", "response_model", "absent_usage", "tool_call", "image_input", + } + ), + "bedrock_converse": frozenset( + { + "cache_read", "cache_write_5m", "cache_write_1h", "absent_usage", + "tool_call", "image_input", + } + ), + "vertex_generate": frozenset( + { + "cache_read", "reasoning", "audio", "web_search", "response_model", + "absent_usage", "tool_call", "image_input", "prompt_blocked", + } + ), }) CaseName: TypeAlias = Literal[ @@ -270,6 +399,7 @@ class Case: scenario_id=scenario_id, wire=model.wire, usage=self.usage, + model=model.provider_model, output=ScriptedOutput( text=text, response_model=model.override_model if self.response_model_override else None, @@ -296,7 +426,9 @@ _PROMPT_BLOCKED_USAGE: Final = ScriptedUsage(fresh_input_tokens=1000, output_tok def _web_search_case(model: FrontierModel) -> Case: - counts_exactly: Final = model.wire in ("openai_responses", "anthropic_messages", "gemini_generate") + counts_exactly: Final = model.wire in ( + "openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate" + ) return Case( name="web_search", usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), @@ -611,12 +743,12 @@ def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: wire's normalization: Anthropic folds cache read/write into prompt_tokens, everyone else reports the totals the wire emitted.""" u: Final = case.usage - if model.wire == "anthropic_messages": + if model.wire in ("anthropic_messages", "bedrock_converse"): return ( u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, u.output_tokens, ) - if model.wire == "gemini_generate": + if model.wire in ("gemini_generate", "vertex_generate"): return ( u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index e1a6c430307..00230fabeba 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -15,10 +15,15 @@ Layout on one port: - ``GET /health`` liveness - ``POST /_scenarios`` register a Scenario JSON, returns its id - ``DELETE /_scenarios/`` remove it +- ``POST /_oauth/token`` fake Google OAuth token endpoint for the + Vertex service-account credential's refresh call - ``POST ///`` provider wire; mount is one of - ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks`` and the - remainder is whatever path the provider client appends (``chat/completions``, - ``responses``, ``v1/messages``, ``models/:generateContent`` ...) + ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks``, ``azure``, + ``bedrock``, ``vertex`` and the remainder is whatever path the provider + client appends (``chat/completions``, ``responses``, ``v1/messages``, + ``models/:generateContent`` ...). Vertex appends ``:generateContent`` / + ``:streamGenerateContent`` to the mount segment itself, and Bedrock Converse + targets ``model//converse`` / ``converse-stream`` A request carrying ``"stream": true`` (or the ``:streamGenerateContent`` Gemini verb) gets an SSE answer; ``stream_usage`` on the Scenario decides whether the @@ -28,15 +33,17 @@ final stream chunk carries usage or the provider reports none. from __future__ import annotations import json +import struct import sys import threading import time +import zlib from collections.abc import Mapping from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from types import MappingProxyType from typing import Final, Literal, TypeAlias -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, model_validator @@ -47,6 +54,9 @@ Wire: TypeAlias = Literal[ "gemini_generate", "together_chat", "fireworks_chat", + "azure_chat", + "bedrock_converse", + "vertex_generate", ] WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( @@ -57,6 +67,9 @@ WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( "gemini_generate": "gemini", "together_chat": "together", "fireworks_chat": "fireworks", + "azure_chat": "azure", + "bedrock_converse": "bedrock", + "vertex_generate": "vertex", } ) @@ -69,6 +82,7 @@ _TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( { "openai_responses": frozenset({"incomplete", "unvalidated"}), "gemini_generate": frozenset({"prompt_blocked"}), + "vertex_generate": frozenset({"prompt_blocked"}), } ) @@ -131,6 +145,10 @@ class Scenario(BaseModel): wire: Wire usage: ScriptedUsage output: ScriptedOutput + # The bare provider-facing model name the renderer echoes when the request + # carries no model of its own (Vertex and Bedrock name the model in the URL + # path, not the body). + model: str stream_usage: StreamUsage = "final_chunk" service_tier: ServiceTier | None = None @@ -904,7 +922,208 @@ def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: ) -def _render(scenario: Scenario, *, stream: bool, requested_model: str) -> RenderedResponse: +def _bedrock_usage(u: ScriptedUsage) -> Mapping[str, object]: + # Converse reports uncached input in inputTokens and rides cache reads and + # writes on top-level fields; totalTokens covers every input kind + output. + cache_writes: Final = u.cache_write_5m_tokens + u.cache_write_1h_tokens + return _jobj_opt( + ("inputTokens", u.fresh_input_tokens), + ("outputTokens", u.output_tokens), + ( + "totalTokens", + u.fresh_input_tokens + u.cache_read_tokens + cache_writes + u.output_tokens, + ), + ("cacheReadInputTokens", u.cache_read_tokens) if u.cache_read_tokens else None, + ("cacheWriteInputTokens", cache_writes) if cache_writes else None, + ( + ( + "cacheDetails", + tuple( + _jobj(("inputTokens", count), ("ttl", ttl)) + for count, ttl in ( + (u.cache_write_5m_tokens, "5m"), + (u.cache_write_1h_tokens, "1h"), + ) + if count + ), + ) + if cache_writes + else None + ), + ) + + +def _bedrock_stop_reason(scenario: Scenario) -> str: + if scenario.output.tool_call is not None: + return "tool_use" + return "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason + + +def _bedrock_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: + tool_call: Final = scenario.output.tool_call + if tool_call is not None: + return ( + _jobj( + ( + "toolUse", + _jobj( + ("toolUseId", f"tooluse_{scenario.scenario_id}"), + ("name", tool_call.name), + ("input", json.loads(tool_call.arguments)), + ), + ), + ), + ) + return (_jobj(("text", scenario.output.text)),) + + +def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: + return _jobj( + ( + "output", + _jobj( + ( + "message", + _jobj( + ("role", "assistant"), + ("content", _bedrock_content(scenario)), + ), + ), + ), + ), + ("stopReason", _bedrock_stop_reason(scenario)), + ("usage", _bedrock_usage(scenario.usage)), + ("metrics", _jobj(("latencyMs", 42))), + ) + + +def _aws_event_frame(event_type: str, payload: Mapping[str, object]) -> bytes: + """One application/vnd.amazon.eventstream frame: prelude + prelude CRC32 + + headers + JSON payload + message CRC32, matching botocore EventStreamBuffer.""" + try: + from botocore.eventstream import crc32 as _crc32 + except ImportError: + _crc32 = zlib.crc32 + + def _str_header(name: str, value: str) -> bytes: + name_b: Final = name.encode() + value_b: Final = value.encode() + return ( + struct.pack("!B", len(name_b)) + + name_b + + struct.pack("!B", 7) + + struct.pack("!H", len(value_b)) + + value_b + ) + + payload_bytes: Final = json.dumps(payload, default=dict, separators=(",", ":")).encode() + headers_bytes: Final = ( + _str_header(":event-type", event_type) + + _str_header(":content-type", "application/json") + + _str_header(":message-type", "event") + ) + total_length: Final = 12 + len(headers_bytes) + len(payload_bytes) + 4 + prelude: Final = struct.pack("!II", total_length, len(headers_bytes)) + prelude_crc: Final = struct.pack("!I", _crc32(prelude) & 0xFFFFFFFF) + message: Final = prelude + prelude_crc + headers_bytes + payload_bytes + return message + struct.pack("!I", _crc32(message, 0) & 0xFFFFFFFF) + + +def _bedrock_eventstream(scenario: Scenario) -> bytes: + tool_call: Final = scenario.output.tool_call + block_start: Final[tuple[bytes, ...]] = ( + ( + _aws_event_frame( + "contentBlockStart", + _jobj( + ( + "start", + _jobj( + ( + "toolUse", + _jobj( + ("toolUseId", f"tooluse_{scenario.scenario_id}"), + ("name", tool_call.name), + ), + ), + ), + ), + ("contentBlockIndex", 0), + ), + ), + ) + if tool_call is not None + else () + ) + deltas: Final[tuple[bytes, ...]] = ( + tuple( + _aws_event_frame( + "contentBlockDelta", + _jobj( + ("delta", _jobj(("toolUse", _jobj(("input", arguments_slice))))), + ("contentBlockIndex", 0), + ), + ) + for arguments_slice in _split_arguments(tool_call.arguments) + ) + if tool_call is not None + else ( + _aws_event_frame( + "contentBlockDelta", + _jobj( + ("delta", _jobj(("text", scenario.output.text))), + ("contentBlockIndex", 0), + ), + ), + ) + ) + return b"".join( + ( + _aws_event_frame("messageStart", _jobj(("role", "assistant"))), + *block_start, + *deltas, + _aws_event_frame("contentBlockStop", _jobj(("contentBlockIndex", 0))), + _aws_event_frame("messageStop", _jobj(("stopReason", _bedrock_stop_reason(scenario)))), + *( + ( + _aws_event_frame( + "metadata", + _jobj( + ("usage", _bedrock_usage(scenario.usage)), + ("metrics", _jobj(("latencyMs", 42))), + ), + ), + ) + if scenario.stream_usage == "final_chunk" + else () + ), + ) + ) + + +def _render( + scenario: Scenario, *, stream: bool, requested_model: str, path_tail: str +) -> RenderedResponse: + # Azure bridges gpt-5.4+ chat requests carrying function tools onto the + # Responses API, which lands on the same mount at openai/responses. + if scenario.wire == "azure_chat" and path_tail.endswith("openai/responses"): + if stream: + return RenderedResponse( + 200, "text/event-stream", _responses_sse(scenario, requested_model) + ) + return RenderedResponse( + 200, "application/json", _json_bytes(_responses_body(scenario, requested_model)) + ) + if scenario.wire == "bedrock_converse": + if stream: + return RenderedResponse( + 200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario) + ) + return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario))) + if scenario.wire == "vertex_generate": + if stream: + return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) if scenario.wire == "anthropic_messages": if stream: return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) @@ -917,7 +1136,8 @@ def _render(scenario: Scenario, *, stream: bool, requested_model: str) -> Render if stream: return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) - # openai_chat, together_chat, fireworks_chat share the OpenAI chat shape. + # openai_chat, together_chat, fireworks_chat and azure_chat share the + # OpenAI chat shape. if stream: return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) @@ -954,17 +1174,28 @@ def _request_body(body: bytes) -> Mapping[str, object]: return MappingProxyType({}) -def _request_wants_stream(path_tail: str, body: bytes) -> bool: - if ":streamGenerateContent" in path_tail: +def _request_wants_stream(mount_endpoint: str | None, path_tail: str, body: bytes) -> bool: + if mount_endpoint == "streamGenerateContent" or ":streamGenerateContent" in path_tail: + return True + if path_tail.endswith("converse-stream"): return True if not body: return False return _request_body(body).get("stream") is True -def _request_model(body: bytes) -> str: +def _request_model(body: bytes, path_tail: str, scenario: Scenario) -> str: model: Final = _request_body(body).get("model") - return model if isinstance(model, str) else "unknown" + if isinstance(model, str): + return model + # Bedrock Converse names the model in the path: model//converse[-stream]. + if path_tail.startswith("model/"): + path_model: Final = path_tail.split("/", 2)[1] if path_tail.count("/") >= 2 else "" + if path_model: + return unquote(path_model) + # Vertex names it in the URL too, but the mount segment swallowed it when + # the api_base carried a path; fall back to the scenario's declared model. + return scenario.model def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: @@ -972,6 +1203,22 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte segments: Final = tuple(segment for segment in path.split("/") if segment) if method == "GET" and segments == ("health",): return RenderedResponse(200, "application/json", _json_bytes(_jobj(("status", "ok")))) + if segments and segments[0] == "_oauth": + if method == "POST" and segments == ("_oauth", "token"): + return RenderedResponse( + 200, + "application/json", + _json_bytes( + _jobj( + ("access_token", "scripted-token"), + ("token_type", "Bearer"), + ("expires_in", 3600), + ) + ), + ) + return RenderedResponse( + 404, "application/json", _json_bytes(_jobj(("error", "unknown control route"))) + ) if segments and segments[0] == "_scenarios": if method == "POST" and len(segments) == 1: try: @@ -998,7 +1245,15 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte return RenderedResponse( 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) ) - scenario_id, mount = segments[0], segments[1] + scenario_id: Final = segments[0] + # Vertex builds {api_base}:{endpoint}, so the mount segment can carry a + # :generateContent / :streamGenerateContent suffix. + mount_segment: Final = segments[1] + mount, mount_endpoint = ( + mount_segment.split(":", 1) + if ":" in mount_segment + else (mount_segment, None) + ) found: Final = store.get(scenario_id) if found is None: return RenderedResponse( @@ -1013,7 +1268,12 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte ), ) tail: Final = "/".join(segments[2:]) - return _render(found, stream=_request_wants_stream(tail, body), requested_model=_request_model(body)) + return _render( + found, + stream=_request_wants_stream(mount_endpoint, tail, body), + requested_model=_request_model(body, tail, found), + path_tail=tail, + ) class _ScriptedHandler(BaseHTTPRequestHandler): diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py index 4da7b31a6ef..a36bb1a8662 100644 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ b/tests/e2e/cost_calculation/test_wire_formats_e2e.py @@ -94,6 +94,40 @@ _WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ "fireworks_ai/kimi-k3", ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), ), + "azure_chat": ( + "azure/gpt-5.6", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), + "bedrock_converse": ( + "anthropic.claude-sonnet-5-v1:0", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + cache_write_5m_tokens=20, + cache_write_1h_tokens=10, + output_tokens=25, + ), + ), + "vertex_generate": ( + "gemini-3.8-flash", + ScriptedUsage( + fresh_input_tokens=80, + cache_read_tokens=40, + output_tokens=25, + reasoning_tokens=15, + audio_input_tokens=5, + audio_output_tokens=3, + ), + ), }) _SHAPE_USAGE: Final = ScriptedUsage(fresh_input_tokens=80, output_tokens=25) @@ -141,6 +175,36 @@ _SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( response_model_override=True, ), ), + ( + "vertex_prompt_blocked", + "vertex_generate", + Case( + name="prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + terminal="prompt_blocked", + response_model_override=True, + ), + ), + ( + "vertex_prompt_blocked_stream", + "vertex_generate", + Case( + name="stream_prompt_blocked", + usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), + stream=True, + terminal="prompt_blocked", + response_model_override=True, + ), + ), + ( + "azure_served_model_override", + "azure_chat", + Case( + name="response_model_override", + usage=_SHAPE_USAGE, + response_model_override=True, + ), + ), ) diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index 68d840870c9..4fba337b701 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -304,6 +304,149 @@ "supports_reasoning": true, "supports_web_search": true }, + "anthropic.claude-sonnet-5-v1:0": { + "cache_creation_input_token_cost": 0.00051, + "cache_creation_input_token_cost_above_1hr": 0.00068, + "cache_read_input_token_cost": 1.7e-05, + "input_cost_per_token": 0.00017, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00034, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-5.4-mini": { + "cache_creation_input_token_cost": 0.00048, + "cache_creation_input_token_cost_above_1hr": 0.00064, + "cache_read_input_token_cost": 1.6e-05, + "input_cost_per_audio_token": 0.00096, + "input_cost_per_token": 0.00016, + "input_cost_per_token_above_200k_tokens": 0.00128, + "input_cost_per_token_flex": 0.00024, + "input_cost_per_token_priority": 0.000272, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00112, + "output_cost_per_reasoning_token": 0.0008, + "output_cost_per_token": 0.00032, + "output_cost_per_token_above_200k_tokens": 0.00144, + "output_cost_per_token_flex": 0.0004, + "output_cost_per_token_priority": 0.000432, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "azure/gpt-5.6": { + "cache_creation_input_token_cost": 0.00044999999999999996, + "cache_creation_input_token_cost_above_1hr": 0.0006000000000000001, + "cache_read_input_token_cost": 1.5e-05, + "input_cost_per_audio_token": 0.0009000000000000001, + "input_cost_per_token": 0.00015000000000000001, + "input_cost_per_token_above_200k_tokens": 0.0012000000000000001, + "input_cost_per_token_flex": 0.000225, + "input_cost_per_token_priority": 0.000255, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0010500000000000002, + "output_cost_per_reasoning_token": 0.00075, + "output_cost_per_token": 0.00030000000000000003, + "output_cost_per_token_above_200k_tokens": 0.00135, + "output_cost_per_token_flex": 0.000375, + "output_cost_per_token_priority": 0.00040499999999999996, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 2.1e-05, + "input_cost_per_audio_token": 0.00126, + "input_cost_per_token": 0.00021, + "input_cost_per_token_above_200k_tokens": 0.00168, + "input_cost_per_token_flex": 0.000315, + "input_cost_per_token_priority": 0.000357, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00147, + "output_cost_per_reasoning_token": 0.0010500000000000002, + "output_cost_per_token": 0.00042, + "output_cost_per_token_above_200k_tokens": 0.0018900000000000001, + "output_cost_per_token_flex": 0.000525, + "output_cost_per_token_priority": 0.000567, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini-3.8-flash": { + "cache_read_input_token_cost": 2e-05, + "input_cost_per_audio_token": 0.0012, + "input_cost_per_token": 0.0002, + "input_cost_per_token_above_200k_tokens": 0.0016, + "input_cost_per_token_flex": 0.0003, + "input_cost_per_token_priority": 0.00034, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0014000000000000002, + "output_cost_per_reasoning_token": 0.001, + "output_cost_per_token": 0.0004, + "output_cost_per_token_above_200k_tokens": 0.0018000000000000001, + "output_cost_per_token_flex": 0.0005, + "output_cost_per_token_priority": 0.00054, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "input_cost_per_token": 0.00019, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00038, + "supports_function_calling": true + }, "together_ai/moonshotai/Kimi-K3": { "cache_creation_input_token_cost": 0.00030000000000000003, "cache_creation_input_token_cost_above_1hr": 0.0004, @@ -363,5 +506,20 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_web_search": true + }, + "us.anthropic.claude-opus-5-v1:0": { + "cache_creation_input_token_cost": 0.0005400000000000001, + "cache_creation_input_token_cost_above_1hr": 0.00072, + "cache_read_input_token_cost": 1.8e-05, + "input_cost_per_token": 0.00018, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00036000000000000004, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true } } diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 7101438c5f8..d96478de1c4 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -1000,6 +1000,7 @@ class ModelInfoBody(BaseModel): access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None + base_model: str | None = None class ModelNewBody(BaseModel): From 2466975d290576de9e89a1d5d69c9ca9a6aab1ab Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 22:57:23 +0000 Subject: [PATCH 08/30] test(e2e): clean cost map decimals and simplify scripted wire helpers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/conftest.py | 52 ++-- .../e2e/cost_calculation/scripted_provider.py | 39 ++- tests/e2e/cost_map.json | 270 +++++++++--------- 3 files changed, 177 insertions(+), 184 deletions(-) diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 8c6db7c0010..3f3e9fd9243 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -11,6 +11,7 @@ Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). from __future__ import annotations +import functools import importlib.util import json import sys @@ -21,6 +22,8 @@ from types import ModuleType from typing import Final, Protocol, cast import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa from cost_matrix import Case, FrontierModel from e2e_config import COST_MAP_PROXY_URL, SCRIPTED_PROVIDER_PROXY_BASE @@ -112,33 +115,25 @@ def client() -> CostCalcClient: return CostCalcClient(proxy=proxy) -_vertex_key_pem: str | None = None +@functools.cache +def _vertex_private_key_pem() -> str: + return rsa.generate_private_key(public_exponent=65537, key_size=2048).private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() def _vertex_service_account_json() -> str: """A service-account credential JSON whose token_uri is the sidecar's /_oauth/token route: the proxy's google-auth refresh then gets a scripted - access token without touching Google. One generated RSA key per process.""" - global _vertex_key_pem # mutable-ok: session-scoped key generation cached for reuse - if _vertex_key_pem is None: - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric import rsa - - _vertex_key_pem = ( - rsa.generate_private_key(public_exponent=65537, key_size=2048) - .private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ) - .decode() - ) + access token without touching Google.""" return json.dumps( { "type": "service_account", "project_id": "cc-scripted-project", "private_key_id": "scripted", - "private_key": _vertex_key_pem, + "private_key": _vertex_private_key_pem(), "client_email": "scripted@cc-scripted-project.iam.gserviceaccount.com", "client_id": "0", "auth_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/authorize", @@ -162,20 +157,21 @@ def register_scenario_deployment( handle: Final = register_scenario(scenario) resources.defer(lambda: delete_scenario(handle)) model_name: Final = f"{model.model_name}-{marker}" - extra_params: Final[dict[str, str]] = dict(model.litellm_params) - if model.wire == "vertex_generate": - extra_params["vertex_credentials"] = _vertex_service_account_json() + params: Final = { + "model": model.litellm_model, + "api_key": model.api_key, + "api_base": handle.api_base(), + **model.litellm_params, + **( + {"vertex_credentials": _vertex_service_account_json()} + if model.wire == "vertex_generate" + else {} + ), + } model_id: Final = client.proxy.register_model( ModelNewBody( model_name=model_name, - litellm_params=LiteLLMParamsBody.model_validate( - { - "model": model.litellm_model, - "api_key": model.api_key, - "api_base": handle.api_base(), - **extra_params, - } - ), + litellm_params=LiteLLMParamsBody.model_validate(params), model_info=ModelInfoBody(base_model=model.base_model), ) ) diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 00230fabeba..982132ed8df 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -997,36 +997,33 @@ def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: ) +def _aws_str_header(name: str, value: str) -> bytes: + """One eventstream header: 1-byte name len + name + type-7 marker + value.""" + name_b: Final = name.encode() + value_b: Final = value.encode() + return ( + struct.pack("!B", len(name_b)) + + name_b + + struct.pack("!B", 7) + + struct.pack("!H", len(value_b)) + + value_b + ) + + def _aws_event_frame(event_type: str, payload: Mapping[str, object]) -> bytes: """One application/vnd.amazon.eventstream frame: prelude + prelude CRC32 + headers + JSON payload + message CRC32, matching botocore EventStreamBuffer.""" - try: - from botocore.eventstream import crc32 as _crc32 - except ImportError: - _crc32 = zlib.crc32 - - def _str_header(name: str, value: str) -> bytes: - name_b: Final = name.encode() - value_b: Final = value.encode() - return ( - struct.pack("!B", len(name_b)) - + name_b - + struct.pack("!B", 7) - + struct.pack("!H", len(value_b)) - + value_b - ) - payload_bytes: Final = json.dumps(payload, default=dict, separators=(",", ":")).encode() headers_bytes: Final = ( - _str_header(":event-type", event_type) - + _str_header(":content-type", "application/json") - + _str_header(":message-type", "event") + _aws_str_header(":event-type", event_type) + + _aws_str_header(":content-type", "application/json") + + _aws_str_header(":message-type", "event") ) total_length: Final = 12 + len(headers_bytes) + len(payload_bytes) + 4 prelude: Final = struct.pack("!II", total_length, len(headers_bytes)) - prelude_crc: Final = struct.pack("!I", _crc32(prelude) & 0xFFFFFFFF) + prelude_crc: Final = struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) message: Final = prelude + prelude_crc + headers_bytes + payload_bytes - return message + struct.pack("!I", _crc32(message, 0) & 0xFFFFFFFF) + return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF) def _bedrock_eventstream(scenario: Scenario) -> bytes: diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index 4fba337b701..85cd5ade3d5 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -1,4 +1,79 @@ { + "anthropic.claude-sonnet-5-v1:0": { + "cache_creation_input_token_cost": 0.00051, + "cache_creation_input_token_cost_above_1hr": 0.00068, + "cache_read_input_token_cost": 1.7e-05, + "input_cost_per_token": 0.00017, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 0.00034, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true + }, + "azure/gpt-5.4-mini": { + "cache_creation_input_token_cost": 0.00048, + "cache_creation_input_token_cost_above_1hr": 0.00064, + "cache_read_input_token_cost": 1.6e-05, + "input_cost_per_audio_token": 0.00096, + "input_cost_per_token": 0.00016, + "input_cost_per_token_above_200k_tokens": 0.00128, + "input_cost_per_token_flex": 0.00024, + "input_cost_per_token_priority": 0.000272, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00112, + "output_cost_per_reasoning_token": 0.0008, + "output_cost_per_token": 0.00032, + "output_cost_per_token_above_200k_tokens": 0.00144, + "output_cost_per_token_flex": 0.0004, + "output_cost_per_token_priority": 0.000432, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, + "azure/gpt-5.6": { + "cache_creation_input_token_cost": 0.00045, + "cache_creation_input_token_cost_above_1hr": 0.0006, + "cache_read_input_token_cost": 1.5e-05, + "input_cost_per_audio_token": 0.0009, + "input_cost_per_token": 0.00015, + "input_cost_per_token_above_200k_tokens": 0.0012, + "input_cost_per_token_flex": 0.000225, + "input_cost_per_token_priority": 0.000255, + "litellm_provider": "azure", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00105, + "output_cost_per_reasoning_token": 0.00075, + "output_cost_per_token": 0.0003, + "output_cost_per_token_above_200k_tokens": 0.00135, + "output_cost_per_token_flex": 0.000375, + "output_cost_per_token_priority": 0.000405, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true + }, "claude-haiku-4-5": { "cache_creation_input_token_cost": 0.00021, "cache_creation_input_token_cost_above_1hr": 0.00028000000000000003, @@ -134,6 +209,64 @@ "supports_reasoning": true, "supports_web_search": true }, + "gemini-3.1-pro-preview": { + "cache_read_input_token_cost": 2.1e-05, + "input_cost_per_audio_token": 0.00126, + "input_cost_per_token": 0.00021, + "input_cost_per_token_above_200k_tokens": 0.00168, + "input_cost_per_token_flex": 0.000315, + "input_cost_per_token_priority": 0.000357, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.00147, + "output_cost_per_reasoning_token": 0.00105, + "output_cost_per_token": 0.00042, + "output_cost_per_token_above_200k_tokens": 0.00189, + "output_cost_per_token_flex": 0.000525, + "output_cost_per_token_priority": 0.000567, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, + "gemini-3.8-flash": { + "cache_read_input_token_cost": 2e-05, + "input_cost_per_audio_token": 0.0012, + "input_cost_per_token": 0.0002, + "input_cost_per_token_above_200k_tokens": 0.0016, + "input_cost_per_token_flex": 0.0003, + "input_cost_per_token_priority": 0.00034, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 2000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_audio_token": 0.0014, + "output_cost_per_reasoning_token": 0.001, + "output_cost_per_token": 0.0004, + "output_cost_per_token_above_200k_tokens": 0.0018, + "output_cost_per_token_flex": 0.0005, + "output_cost_per_token_priority": 0.00054, + "search_context_cost_per_query": { + "search_context_size_high": 0.03, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.02 + }, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_web_search": true, + "web_search_billing_unit": "per_query" + }, "gemini/gemini-3.1-pro-preview": { "cache_read_input_token_cost": 9e-06, "input_cost_per_audio_token": 0.00054, @@ -304,139 +437,6 @@ "supports_reasoning": true, "supports_web_search": true }, - "anthropic.claude-sonnet-5-v1:0": { - "cache_creation_input_token_cost": 0.00051, - "cache_creation_input_token_cost_above_1hr": 0.00068, - "cache_read_input_token_cost": 1.7e-05, - "input_cost_per_token": 0.00017, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00034, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true - }, - "azure/gpt-5.4-mini": { - "cache_creation_input_token_cost": 0.00048, - "cache_creation_input_token_cost_above_1hr": 0.00064, - "cache_read_input_token_cost": 1.6e-05, - "input_cost_per_audio_token": 0.00096, - "input_cost_per_token": 0.00016, - "input_cost_per_token_above_200k_tokens": 0.00128, - "input_cost_per_token_flex": 0.00024, - "input_cost_per_token_priority": 0.000272, - "litellm_provider": "azure", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.00112, - "output_cost_per_reasoning_token": 0.0008, - "output_cost_per_token": 0.00032, - "output_cost_per_token_above_200k_tokens": 0.00144, - "output_cost_per_token_flex": 0.0004, - "output_cost_per_token_priority": 0.000432, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "azure/gpt-5.6": { - "cache_creation_input_token_cost": 0.00044999999999999996, - "cache_creation_input_token_cost_above_1hr": 0.0006000000000000001, - "cache_read_input_token_cost": 1.5e-05, - "input_cost_per_audio_token": 0.0009000000000000001, - "input_cost_per_token": 0.00015000000000000001, - "input_cost_per_token_above_200k_tokens": 0.0012000000000000001, - "input_cost_per_token_flex": 0.000225, - "input_cost_per_token_priority": 0.000255, - "litellm_provider": "azure", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.0010500000000000002, - "output_cost_per_reasoning_token": 0.00075, - "output_cost_per_token": 0.00030000000000000003, - "output_cost_per_token_above_200k_tokens": 0.00135, - "output_cost_per_token_flex": 0.000375, - "output_cost_per_token_priority": 0.00040499999999999996, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "gemini-3.1-pro-preview": { - "cache_read_input_token_cost": 2.1e-05, - "input_cost_per_audio_token": 0.00126, - "input_cost_per_token": 0.00021, - "input_cost_per_token_above_200k_tokens": 0.00168, - "input_cost_per_token_flex": 0.000315, - "input_cost_per_token_priority": 0.000357, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.00147, - "output_cost_per_reasoning_token": 0.0010500000000000002, - "output_cost_per_token": 0.00042, - "output_cost_per_token_above_200k_tokens": 0.0018900000000000001, - "output_cost_per_token_flex": 0.000525, - "output_cost_per_token_priority": 0.000567, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, - "gemini-3.8-flash": { - "cache_read_input_token_cost": 2e-05, - "input_cost_per_audio_token": 0.0012, - "input_cost_per_token": 0.0002, - "input_cost_per_token_above_200k_tokens": 0.0016, - "input_cost_per_token_flex": 0.0003, - "input_cost_per_token_priority": 0.00034, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.0014000000000000002, - "output_cost_per_reasoning_token": 0.001, - "output_cost_per_token": 0.0004, - "output_cost_per_token_above_200k_tokens": 0.0018000000000000001, - "output_cost_per_token_flex": 0.0005, - "output_cost_per_token_priority": 0.00054, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, "meta.llama4-maverick-17b-instruct-v1:0": { "input_cost_per_token": 0.00019, "litellm_provider": "bedrock_converse", @@ -508,7 +508,7 @@ "supports_web_search": true }, "us.anthropic.claude-opus-5-v1:0": { - "cache_creation_input_token_cost": 0.0005400000000000001, + "cache_creation_input_token_cost": 0.00054, "cache_creation_input_token_cost_above_1hr": 0.00072, "cache_read_input_token_cost": 1.8e-05, "input_cost_per_token": 0.00018, @@ -517,7 +517,7 @@ "max_output_tokens": 128000, "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 0.00036000000000000004, + "output_cost_per_token": 0.00036, "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true From 813d96f26ea6780bacb4b5ad562f1aecc3cb5069 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 16 Sep 2026 23:18:26 +0000 Subject: [PATCH 09/30] fix(e2e): resolve remaining merge markers in e2e_config Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/e2e_config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index b34eadd8744..e19cfaa684f 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -145,7 +145,6 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK" REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" -<<<<<<< HEAD # The cost_calculation suite needs a proxy booted with LITELLM_MODEL_COST_MAP_URL # pointing at tests/e2e/cost_map.json (its whole map is test-owned rates) plus a # scripted-provider sidecar; deselected unless the opt-in env var is set. @@ -162,9 +161,6 @@ SCRIPTED_PROVIDER_CONTROL_URL = os.environ.get( SCRIPTED_PROVIDER_PROXY_BASE = os.environ.get( "E2E_SCRIPTED_PROVIDER_PROXY_BASE", SCRIPTED_PROVIDER_CONTROL_URL ).rstrip("/") -||||||| 930ec9643a -======= -CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) From bdfff602fb0325f88768f7c4411cce921ab28fcb Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 00:47:55 +0000 Subject: [PATCH 10/30] test(e2e): drive the cost matrix from cases.json and expected.json goldens Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/cost_calculation/cases.json | 231 ++ tests/e2e/cost_calculation/conftest.py | 10 +- tests/e2e/cost_calculation/cost_matrix.py | 788 ++----- tests/e2e/cost_calculation/expected.json | 2004 +++++++++++++++++ .../e2e/cost_calculation/generate_expected.py | 189 ++ .../e2e/cost_calculation/test_matrix_data.py | 64 + .../test_token_pricing_e2e.py | 62 +- .../cost_calculation/test_wire_formats_e2e.py | 368 --- 9 files changed, 2761 insertions(+), 957 deletions(-) create mode 100644 tests/e2e/cost_calculation/cases.json create mode 100644 tests/e2e/cost_calculation/expected.json create mode 100644 tests/e2e/cost_calculation/generate_expected.py create mode 100644 tests/e2e/cost_calculation/test_matrix_data.py delete mode 100644 tests/e2e/cost_calculation/test_wire_formats_e2e.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 49cfc29aa17..707d35b4aa6 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; asserts literal rate arithmetic on scripted usage across every provider wire and pricing component, deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json new file mode 100644 index 00000000000..e898557ea35 --- /dev/null +++ b/tests/e2e/cost_calculation/cases.json @@ -0,0 +1,231 @@ +{ + "deployments": [ + { + "map_key": "azure/gpt-5.4-mini", + "litellm_model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + } + ], + "cases": [ + { + "name": "basic", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40} + }, + { + "name": "cache_read", + "usage": {"fresh_input_tokens": 100, "cache_read_tokens": 50, "output_tokens": 30}, + "requires_rates": ["cache_read_input_token_cost"], + "requires_caps": ["cache_read"] + }, + { + "name": "cache_write_5m", + "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 60, "output_tokens": 30}, + "requires_rates": ["cache_creation_input_token_cost"], + "requires_caps": ["cache_write_5m"] + }, + { + "name": "cache_write_1h", + "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 40, "output_tokens": 30}, + "requires_rates": ["cache_creation_input_token_cost_above_1hr", "cache_creation_input_token_cost"], + "requires_caps": ["cache_write_1h"] + }, + { + "name": "reasoning", + "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "reasoning_tokens": 70}, + "requires_rates": ["output_cost_per_reasoning_token"], + "requires_caps": ["reasoning"] + }, + { + "name": "audio", + "usage": {"fresh_input_tokens": 100, "audio_input_tokens": 25, "output_tokens": 30, "audio_output_tokens": 15}, + "requires_rates": ["input_cost_per_audio_token", "output_cost_per_audio_token"], + "requires_caps": ["audio"] + }, + { + "name": "tiered", + "usage": {"fresh_input_tokens": 200001, "output_tokens": 30}, + "requires_rates": ["input_cost_per_token_above_200k_tokens", "output_cost_per_token_above_200k_tokens"] + }, + { + "name": "service_tier_flex", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "service_tier": "flex", + "requires_rates": ["input_cost_per_token_flex", "output_cost_per_token_flex"] + }, + { + "name": "service_tier_priority", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "service_tier": "priority", + "requires_rates": ["input_cost_per_token_priority", "output_cost_per_token_priority"] + }, + { + "name": "web_search", + "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 3}, + "requires_rates": ["search_context_cost_per_query"], + "requires_caps": ["web_search"] + }, + { + "name": "stream", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true + }, + { + "name": "stream_no_usage", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "exact_spend": false, + "requires_caps": ["absent_usage"] + }, + { + "name": "response_model_override", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "response_model_override": true, + "requires_caps": ["response_model"] + }, + { + "name": "stream_response_model_override", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "response_model_override": true, + "requires_caps": ["response_model"] + }, + { + "name": "tool_call", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "tool_call": true, + "requires_caps": ["tool_call"] + }, + { + "name": "stream_tool_call", + "usage": {"fresh_input_tokens": 80, "output_tokens": 25}, + "stream": true, + "tool_call": true, + "requires_caps": ["tool_call"] + }, + { + "name": "stream_no_usage_tool_call", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "tool_call": true, + "exact_spend": false, + "requires_caps": ["absent_usage", "tool_call"] + }, + { + "name": "stream_no_usage_image_input", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "image_input": true, + "exact_spend": false, + "requires_caps": ["absent_usage", "image_input"] + }, + { + "name": "stream_incomplete", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "terminal": "incomplete", + "requires_caps": ["responses_terminal"] + }, + { + "name": "stream_no_usage_incomplete", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "terminal": "incomplete", + "exact_spend": false, + "requires_caps": ["responses_terminal"] + }, + { + "name": "stream_unvalidated", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "terminal": "unvalidated", + "requires_caps": ["responses_terminal"] + }, + { + "name": "stream_no_usage_unvalidated", + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "stream": true, + "stream_usage": "absent", + "terminal": "unvalidated", + "exact_spend": false, + "requires_caps": ["responses_terminal"] + }, + { + "name": "prompt_blocked", + "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, + "terminal": "prompt_blocked", + "response_model_override": true, + "requires_caps": ["prompt_blocked"] + }, + { + "name": "stream_prompt_blocked", + "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, + "stream": true, + "terminal": "prompt_blocked", + "response_model_override": true, + "requires_caps": ["prompt_blocked"] + }, + { + "name": "all_components_chat", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 10, + "output_tokens": 25, + "reasoning_tokens": 15, + "audio_input_tokens": 5, + "audio_output_tokens": 3 + }, + "wires": ["openai_chat", "azure_chat", "together_chat"] + }, + { + "name": "all_components_fireworks", + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, + "wires": ["fireworks_chat"] + }, + { + "name": "all_components_anthropic", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 10, + "output_tokens": 25 + }, + "wires": ["anthropic_messages", "bedrock_converse"] + }, + { + "name": "all_components_anthropic_stream", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "cache_write_5m_tokens": 20, + "cache_write_1h_tokens": 10, + "output_tokens": 25 + }, + "stream": true, + "wires": ["anthropic_messages"] + }, + { + "name": "all_components_gemini", + "usage": { + "fresh_input_tokens": 80, + "cache_read_tokens": 40, + "output_tokens": 25, + "reasoning_tokens": 15, + "audio_input_tokens": 5, + "audio_output_tokens": 3 + }, + "wires": ["gemini_generate", "vertex_generate"] + }, + { + "name": "all_components_responses", + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, + "wires": ["openai_responses"] + } + ] +} diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 3f3e9fd9243..3de9786854e 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -1,10 +1,12 @@ """Cost-calculation suite fixtures. Runs against a dedicated proxy whose whole model cost map is the test-owned -``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL), so every deployment -bills at rates the test asserts literal arithmetic on. Provider calls are -answered by the scripted-provider sidecar (``scripted_provider.py``), registered -per scenario over its control API. +``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL); every map entry is a +deployment under test, the request shapes live in ``cases.json``, and the +asserted goldens live in ``expected.json`` (regenerate proposals with +``generate_expected.py``). Provider calls are answered by the +scripted-provider sidecar (``scripted_provider.py``), registered per scenario +over its control API. Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). """ diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 37495f37b0b..b03c851d208 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -1,18 +1,16 @@ -"""The cost-calculation matrix: frontier model set, the pricing-component cases -each model runs, and the expected-cost arithmetic. +"""The cost-calculation matrix: the model set derived from the test cost map, +the request/response cases from ``cases.json``, and the loaders both use. -Rates come from ``tests/e2e/cost_map.json``, which the proxy under test loads as -its ENTIRE model cost map (LITELLM_MODEL_COST_MAP_URL), so an entry's rates are -exactly what the proxy bills and nothing in the suite depends on the bundled -map. Each model's rates are a distinct multiple of a shared base set, so a -component billed at the wrong model's rate (or the wrong case's rate) can never -coincidentally match. - -Case applicability is pricing-field-gated AND wire-gated: a case runs for a -model only when the entry carries the rate the case exercises and the wire can -report the token kind that rate prices. When the wire cannot report a kind -(e.g. Anthropic has no reasoning-token field, Responses reports no cache -creation), the case is absent from the matrix rather than silently zero. +Three data files drive the suite; nothing in Python lists models or cases: +- ``tests/e2e/cost_map.json`` is the proxy's ENTIRE model cost map + (LITELLM_MODEL_COST_MAP_URL); every entry becomes a deployment under test. +- ``tests/e2e/cost_calculation/cases.json`` is the case list; each case runs + for a model when the entry carries the rates it exercises (``requires_rates``) + and the wire can report the token kinds involved (``requires_caps`` / + ``wires``). +- ``tests/e2e/cost_calculation/expected.json`` holds the reviewed goldens; the + tests assert them verbatim and never compute a price themselves. The rate + arithmetic that proposes goldens lives in ``generate_expected.py``, not here. """ from __future__ import annotations @@ -26,13 +24,15 @@ from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Final, Literal from pydantic import BaseModel, ConfigDict, TypeAdapter from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" +CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" +EXPECTED_PATH: Final = Path(__file__).resolve().parent / "expected.json" class SearchContextCostPerQuery(BaseModel): @@ -77,119 +77,90 @@ _COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( TIER_THRESHOLD_TOKENS: Final = 200_000 -@dataclass(frozen=True, slots=True) -class FrontierModel: - """One deployment under test: the model_name the suite registers, the - provider-prefixed litellm model string, the wire the scripted upstream - speaks, its cost-map key, and the sibling map model the response_model - override case reports.""" +class DeploymentSpec(BaseModel): + """A deployment-level fact from cases.json: when a map key needs a + registered deployment name that is not its provider model (or a + model_info.base_model pin), the matrix uses these instead of the defaults.""" + + model_config = ConfigDict(frozen=True) - model_name: str - litellm_model: str - wire: Wire map_key: str - override_model: str | None = None - override_map_key: str | None = None - # Registered as model_info.base_model; when set, the provider-reported - # model loses to it and every case bills at this deployment's own rates. + litellm_model: str | None = None base_model: str | None = None - # Extra litellm_params merged into the /model/new registration (api_version, - # aws_* credentials, vertex_* auth). - litellm_params: Mapping[str, str] = MappingProxyType({}) - - @property - def rates(self) -> CostMapEntry: - return _COST_MAP[self.map_key] - - @property - def override_rates(self) -> CostMapEntry: - if self.base_model is not None or self.override_map_key is None: - return self.rates - return _COST_MAP[self.override_map_key] - - @property - def provider_model(self) -> str: - """The bare provider-facing model name: litellm_model minus the provider - prefix and any routing segment (converse/, responses/).""" - tail: Final = self.litellm_model.split("/")[1:] - return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) - - @property - def provider(self) -> str: - return self.rates.litellm_provider - - @property - def api_key(self) -> str: - # The scripted upstream ignores auth; a fixed bogus key proves the suite - # spends zero real provider calls. - return "sk-scripted-provider" -# Response-model override targets: emit a sibling's bare provider-facing name so -# the biller's provider-prefixed lookup lands on that sibling's map key. -_OVERRIDE_MODELS: Final[Mapping[str, str]] = MappingProxyType({ - "gpt-5.6": "gpt-5.4-mini", - "gpt-5.5-pro": "gpt-5.3-codex", - "gpt-5.3-codex": "gpt-5.5-pro", - "gpt-5.4-mini": "gpt-5.6", - "claude-opus-5": "claude-sonnet-5", - "claude-sonnet-5": "claude-opus-5", - "claude-haiku-4-5": "claude-sonnet-5", - "gemini/gemini-3.8-flash": "gemini-3.1-pro-preview", - "gemini/gemini-3.1-pro-preview": "gemini-3.8-flash", - "together_ai/moonshotai/Kimi-K3": "zai-org/GLM-5.3", - "together_ai/zai-org/GLM-5.3": "moonshotai/Kimi-K3", - "fireworks_ai/kimi-k3": "qwen3p8-max", - "fireworks_ai/qwen3p8-max": "kimi-k3", - "fireworks_ai/deepseek-v4p1-flash": "kimi-k3", -}) +class Case(BaseModel): + """One request/response shape from cases.json; gated onto a model by + ``requires_rates`` (entry must carry each rate field), ``requires_caps`` + (the wire must report the token kind) and ``wires`` (shape is wire-specific).""" -_OVERRIDE_MAP_KEYS: Final[Mapping[str, str]] = MappingProxyType({ - "gpt-5.4-mini": "gpt-5.4-mini", - "gpt-5.6": "gpt-5.6", - "gpt-5.3-codex": "gpt-5.3-codex", - "gpt-5.5-pro": "gpt-5.5-pro", - "claude-sonnet-5": "claude-sonnet-5", - "claude-opus-5": "claude-opus-5", - "gemini-3.1-pro-preview": "gemini/gemini-3.1-pro-preview", - "gemini-3.8-flash": "gemini/gemini-3.8-flash", - "zai-org/GLM-5.3": "together_ai/zai-org/GLM-5.3", - "moonshotai/Kimi-K3": "together_ai/moonshotai/Kimi-K3", - "qwen3p8-max": "fireworks_ai/qwen3p8-max", - "kimi-k3": "fireworks_ai/kimi-k3", -}) + model_config = ConfigDict(frozen=True) + + name: str + usage: ScriptedUsage + stream: bool = False + stream_usage: Literal["final_chunk", "absent"] = "final_chunk" + service_tier: Literal["flex", "priority"] | None = None + response_model_override: bool = False + exact_spend: bool = True + tool_call: bool = False + image_input: bool = False + terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" + requires_rates: tuple[str, ...] = () + requires_caps: tuple[str, ...] = () + wires: tuple[Wire, ...] | None = None + + def applies_to(self, model: FrontierModel) -> bool: + if self.wires is not None and model.wire not in self.wires: + return False + caps: Final = _WIRE_CAPS[model.wire] + if not frozenset(self.requires_caps) <= caps: + return False + return all( + getattr(model.rates, field, None) is not None for field in self.requires_rates + ) + + def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: + return Scenario( + scenario_id=scenario_id, + wire=model.wire, + usage=self.usage, + model=model.provider_model, + output=ScriptedOutput( + text=text, + response_model=model.override_model if self.response_model_override else None, + tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) + if self.tool_call + else None, + terminal=self.terminal, + ), + stream_usage=self.stream_usage, + service_tier=self.service_tier, + ) -_FRONTIER_SPECS: Final[tuple[tuple[str, str, Wire], ...]] = ( - ("gpt-5.6", "openai/gpt-5.6", "openai_chat"), - ("gpt-5.5-pro", "openai/gpt-5.5-pro", "openai_responses"), - ("gpt-5.3-codex", "openai/gpt-5.3-codex", "openai_responses"), - ("gpt-5.4-mini", "openai/gpt-5.4-mini", "openai_chat"), - ("claude-opus-5", "anthropic/claude-opus-5", "anthropic_messages"), - ("claude-sonnet-5", "anthropic/claude-sonnet-5", "anthropic_messages"), - ("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "anthropic_messages"), - ("gemini/gemini-3.8-flash", "gemini/gemini-3.8-flash", "gemini_generate"), - ("gemini/gemini-3.1-pro-preview", "gemini/gemini-3.1-pro-preview", "gemini_generate"), - ("together_ai/moonshotai/Kimi-K3", "together_ai/moonshotai/Kimi-K3", "together_chat"), - ("together_ai/zai-org/GLM-5.3", "together_ai/zai-org/GLM-5.3", "together_chat"), - ("fireworks_ai/kimi-k3", "fireworks_ai/kimi-k3", "fireworks_chat"), - ("fireworks_ai/qwen3p8-max", "fireworks_ai/qwen3p8-max", "fireworks_chat"), - ("fireworks_ai/deepseek-v4p1-flash", "fireworks_ai/deepseek-v4p1-flash", "fireworks_chat"), +class _CasesFile(BaseModel): + model_config = ConfigDict(frozen=True) + + deployments: tuple[DeploymentSpec, ...] = () + cases: tuple[Case, ...] = () + + +_CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) +CASES: Final[tuple[Case, ...]] = _CASES_FILE.cases +_DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( + {spec.map_key: spec for spec in _CASES_FILE.deployments} ) @dataclass(frozen=True, slots=True) -class _ExtendedSpec: - """A frontier entry whose override target, model_info.base_model or extra - litellm_params can't be derived from the map key alone.""" +class _ProviderWiring: + """How a (litellm_provider, mode) pair maps to a sidecar wire, the provider + prefix on the registered litellm model string, and extra litellm_params.""" - map_key: str - litellm_model: str wire: Wire - override_model: str | None = None - override_map_key: str | None = None - base_model: str | None = None - litellm_params: Mapping[str, str] = MappingProxyType({}) + model_prefix: str | None + litellm_params: Mapping[str, str] _AZURE_PARAMS: Final[Mapping[str, str]] = MappingProxyType({"api_version": "2025-04-01-preview"}) @@ -207,87 +178,134 @@ _VERTEX_PARAMS: Final[Mapping[str, str]] = MappingProxyType( } ) -_EXTENDED_SPECS: Final[tuple[_ExtendedSpec, ...]] = ( - _ExtendedSpec( - map_key="azure/gpt-5.6", - litellm_model="azure/gpt-5.6", - wire="azure_chat", - override_model="gpt-5.4-mini", - override_map_key="azure/gpt-5.4-mini", - litellm_params=_AZURE_PARAMS, - ), - _ExtendedSpec( - # Deployment name is not a model; base_model pins billing so the - # response's model field loses, proving base_model wins. - map_key="azure/gpt-5.4-mini", - litellm_model="azure/cc-pinned-deployment", - wire="azure_chat", - override_model="gpt-5.6", - override_map_key="azure/gpt-5.6", - base_model="azure/gpt-5.4-mini", - litellm_params=_AZURE_PARAMS, - ), - _ExtendedSpec( - map_key="anthropic.claude-sonnet-5-v1:0", - litellm_model="bedrock/converse/anthropic.claude-sonnet-5-v1:0", - wire="bedrock_converse", - litellm_params=_BEDROCK_PARAMS, - ), - _ExtendedSpec( - map_key="us.anthropic.claude-opus-5-v1:0", - litellm_model="bedrock/converse/us.anthropic.claude-opus-5-v1:0", - wire="bedrock_converse", - litellm_params=_BEDROCK_PARAMS, - ), - _ExtendedSpec( - map_key="meta.llama4-maverick-17b-instruct-v1:0", - litellm_model="bedrock/converse/meta.llama4-maverick-17b-instruct-v1:0", - wire="bedrock_converse", - litellm_params=_BEDROCK_PARAMS, - ), - _ExtendedSpec( - map_key="gemini-3.8-flash", - litellm_model="vertex_ai/gemini-3.8-flash", - wire="vertex_generate", - override_model="gemini-3.1-pro-preview", - override_map_key="gemini-3.1-pro-preview", - litellm_params=_VERTEX_PARAMS, - ), - _ExtendedSpec( - map_key="gemini-3.1-pro-preview", - litellm_model="vertex_ai/gemini-3.1-pro-preview", - wire="vertex_generate", - override_model="gemini-3.8-flash", - override_map_key="gemini-3.8-flash", - litellm_params=_VERTEX_PARAMS, - ), +_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = MappingProxyType( + { + ("openai", "chat"): _ProviderWiring("openai_chat", "openai", MappingProxyType({})), + ("openai", "responses"): _ProviderWiring( + "openai_responses", "openai", MappingProxyType({}) + ), + ("anthropic", "chat"): _ProviderWiring( + "anthropic_messages", "anthropic", MappingProxyType({}) + ), + ("gemini", "chat"): _ProviderWiring("gemini_generate", None, MappingProxyType({})), + ("together_ai", "chat"): _ProviderWiring("together_chat", None, MappingProxyType({})), + ("fireworks_ai", "chat"): _ProviderWiring("fireworks_chat", None, MappingProxyType({})), + ("azure", "chat"): _ProviderWiring("azure_chat", None, _AZURE_PARAMS), + ("bedrock_converse", "chat"): _ProviderWiring( + "bedrock_converse", "bedrock/converse", _BEDROCK_PARAMS + ), + ("vertex_ai-language-models", "chat"): _ProviderWiring( + "vertex_generate", "vertex_ai", _VERTEX_PARAMS + ), + } ) +@dataclass(frozen=True, slots=True) +class FrontierModel: + """One deployment under test, derived from a cost-map entry: the model_name + the suite registers, the provider-prefixed litellm model string, the wire + the scripted upstream speaks, and the sibling map model the response_model + override case reports.""" + + model_name: str + litellm_model: str + wire: Wire + map_key: str + override_model: str | None = None + override_map_key: str | None = None + # Registered as model_info.base_model; when set, the provider-reported + # model loses to it and every case bills at this deployment's own rates. + base_model: str | None = None + litellm_params: Mapping[str, str] = MappingProxyType({}) + + @property + def rates(self) -> CostMapEntry: + return _COST_MAP[self.map_key] + + @property + def override_rates(self) -> CostMapEntry: + if self.base_model is not None or self.override_map_key is None: + return self.rates + return _COST_MAP[self.override_map_key] + + @property + def provider_model(self) -> str: + """The bare provider-facing model name: litellm_model minus the provider + prefix and any routing segment (converse/, responses/).""" + return _provider_model(self.litellm_model) + + @property + def provider(self) -> str: + return self.rates.litellm_provider + + @property + def api_key(self) -> str: + # The scripted upstream ignores auth; a fixed bogus key proves the suite + # spends zero real provider calls. + return "sk-scripted-provider" + + +def _provider_model(litellm_model: str) -> str: + tail: Final = litellm_model.split("/")[1:] + return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) + + +def _litellm_model_for(map_key: str, wiring: _ProviderWiring) -> str: + if wiring.model_prefix is None: + return map_key + if map_key.startswith(f"{wiring.model_prefix}/"): + return map_key + return f"{wiring.model_prefix}/{map_key}" + + def _frontier() -> tuple[FrontierModel, ...]: - return tuple( - FrontierModel( - model_name=f"cc-{map_key.replace('/', '-').lower()}", - litellm_model=litellm_model, - wire=wire, - map_key=map_key, - override_model=_OVERRIDE_MODELS[map_key], - override_map_key=_OVERRIDE_MAP_KEYS[_OVERRIDE_MODELS[map_key]], - ) - for map_key, litellm_model, wire in _FRONTIER_SPECS - ) + tuple( - FrontierModel( - model_name=f"cc-{spec.map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", - litellm_model=spec.litellm_model, - wire=spec.wire, - map_key=spec.map_key, - override_model=spec.override_model, - override_map_key=spec.override_map_key, - base_model=spec.base_model, - litellm_params=spec.litellm_params, - ) - for spec in _EXTENDED_SPECS + groups: Final[Mapping[tuple[str, str], tuple[str, ...]]] = MappingProxyType( + { + pair: tuple(sorted(k for k, e in _COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) + for pair in {(e.litellm_provider, e.mode) for e in _COST_MAP.values()} + } ) + models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple + for map_key in sorted(_COST_MAP): + entry: Final = _COST_MAP[map_key] + pair: Final = (entry.litellm_provider, entry.mode) + wiring: Final = _PROVIDER_WIRING.get(pair) + if wiring is None: + raise ValueError( + f"cost_map entry {map_key} has no wiring for " + f"(litellm_provider={pair[0]}, mode={pair[1]}); add a " + f"_ProviderWiring row in cost_matrix.py" + ) + siblings: Final = groups[pair] + override_key: Final = ( + siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None + ) + override_litellm: Final = ( + _litellm_model_for(override_key, wiring) if override_key is not None else None + ) + deployment: Final = _DEPLOYMENTS.get(map_key) + models.append( + FrontierModel( + model_name=f"cc-{map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", + litellm_model=( + deployment.litellm_model + if deployment is not None and deployment.litellm_model is not None + else _litellm_model_for(map_key, wiring) + ), + wire=wiring.wire, + map_key=map_key, + override_model=( + _provider_model(override_litellm) + if override_litellm is not None + else None + ), + override_map_key=override_key, + base_model=deployment.base_model if deployment is not None else None, + litellm_params=wiring.litellm_params, + ) + ) + return tuple(models) FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() @@ -350,71 +368,6 @@ _WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ ), }) -CaseName: TypeAlias = Literal[ - "basic", - "cache_read", - "cache_write_5m", - "cache_write_1h", - "reasoning", - "audio", - "tiered", - "service_tier_flex", - "service_tier_priority", - "web_search", - "stream", - "stream_no_usage", - "response_model_override", - "stream_response_model_override", - "tool_call", - "stream_no_usage_tool_call", - "stream_no_usage_image_input", - "stream_no_usage_incomplete", - "stream_unvalidated", - "stream_no_usage_unvalidated", - "prompt_blocked", - "stream_prompt_blocked", -] - - -@dataclass(frozen=True, slots=True) -class Case: - name: CaseName - usage: ScriptedUsage - stream: bool = False - stream_usage: Literal["final_chunk", "absent"] = "final_chunk" - service_tier: Literal["flex", "priority"] | None = None - # For web_search the wire's reported call count is not always what gets - # billed: chat-completions surfaces only expose url_citation annotations, so - # the biller floors to one call; responses/messages/gemini report a real - # count. - billed_web_search_calls: int = 0 - response_model_override: bool = False - exact_spend: bool = True - tool_call: bool = False - image_input: bool = False - terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" - - def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: - return Scenario( - scenario_id=scenario_id, - wire=model.wire, - usage=self.usage, - model=model.provider_model, - output=ScriptedOutput( - text=text, - response_model=model.override_model if self.response_model_override else None, - tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) - if self.tool_call - else None, - terminal=self.terminal, - ), - stream_usage=self.stream_usage, - service_tier=self.service_tier, - ) - - -_BASIC_USAGE: Final = ScriptedUsage(fresh_input_tokens=120, output_tokens=40) - TOOL_CALL_ARGUMENTS: Final = json.dumps({ "city": "Berlin", "days": 7, @@ -422,284 +375,9 @@ TOOL_CALL_ARGUMENTS: Final = json.dumps({ "notes": "filler " * 30, }) -_PROMPT_BLOCKED_USAGE: Final = ScriptedUsage(fresh_input_tokens=1000, output_tokens=0) - - -def _web_search_case(model: FrontierModel) -> Case: - counts_exactly: Final = model.wire in ( - "openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate" - ) - return Case( - name="web_search", - usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, web_search_calls=3), - billed_web_search_calls=3 if counts_exactly else 1, - ) - def cases_for(model: FrontierModel) -> tuple[Case, ...]: - rates: Final = model.rates - caps: Final = _WIRE_CAPS[model.wire] - candidates: Final[tuple[Case | None, ...]] = ( - Case(name="basic", usage=_BASIC_USAGE), - ( - Case(name="cache_read", usage=ScriptedUsage(fresh_input_tokens=100, cache_read_tokens=50, output_tokens=30)) - if rates.cache_read_input_token_cost is not None and "cache_read" in caps - else None - ), - ( - Case( - name="cache_write_5m", - usage=ScriptedUsage(fresh_input_tokens=90, cache_write_5m_tokens=60, output_tokens=30), - ) - if rates.cache_creation_input_token_cost is not None and "cache_write_5m" in caps - else None - ), - ( - Case( - name="cache_write_1h", - usage=ScriptedUsage( - fresh_input_tokens=90, - cache_write_5m_tokens=20, - cache_write_1h_tokens=40, - output_tokens=30, - ), - ) - if ( - rates.cache_creation_input_token_cost_above_1hr is not None - and rates.cache_creation_input_token_cost is not None - and "cache_write_1h" in caps - ) - else None - ), - ( - Case( - name="reasoning", - usage=ScriptedUsage(fresh_input_tokens=100, output_tokens=30, reasoning_tokens=70), - ) - if rates.output_cost_per_reasoning_token is not None and "reasoning" in caps - else None - ), - ( - Case( - name="audio", - usage=ScriptedUsage( - fresh_input_tokens=100, audio_input_tokens=25, output_tokens=30, audio_output_tokens=15 - ), - ) - if ( - rates.input_cost_per_audio_token is not None - and rates.output_cost_per_audio_token is not None - and "audio" in caps - ) - else None - ), - ( - Case( - name="tiered", - usage=ScriptedUsage( - fresh_input_tokens=TIER_THRESHOLD_TOKENS + 1, output_tokens=30 - ), - ) - if ( - rates.input_cost_per_token_above_200k_tokens is not None - and rates.output_cost_per_token_above_200k_tokens is not None - ) - else None - ), - ( - Case(name="service_tier_flex", usage=_BASIC_USAGE, service_tier="flex") - if rates.input_cost_per_token_flex is not None and rates.output_cost_per_token_flex is not None - else None - ), - ( - Case(name="service_tier_priority", usage=_BASIC_USAGE, service_tier="priority") - if rates.input_cost_per_token_priority is not None and rates.output_cost_per_token_priority is not None - else None - ), - _web_search_case(model) if rates.search_context_cost_per_query is not None and "web_search" in caps else None, - Case(name="stream", usage=_BASIC_USAGE, stream=True), - ( - Case( - name="stream_no_usage", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - exact_spend=False, - ) - if "absent_usage" in caps - else None - ), - ( - Case(name="response_model_override", usage=_BASIC_USAGE, response_model_override=True) - if "response_model" in caps - else None - ), - ( - Case( - name="stream_response_model_override", - usage=_BASIC_USAGE, - stream=True, - response_model_override=True, - ) - if "response_model" in caps - else None - ), - ( - Case(name="tool_call", usage=_BASIC_USAGE, tool_call=True) - if "tool_call" in caps - else None - ), - ( - Case( - name="stream_no_usage_tool_call", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - tool_call=True, - exact_spend=False, - ) - if "absent_usage" in caps and "tool_call" in caps - else None - ), - ( - Case( - name="stream_no_usage_image_input", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - image_input=True, - exact_spend=False, - ) - if "absent_usage" in caps and "image_input" in caps - else None - ), - ( - Case( - name="stream_no_usage_incomplete", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - terminal="incomplete", - exact_spend=False, - ) - if "responses_terminal" in caps - else None - ), - ( - Case( - name="stream_unvalidated", - usage=_BASIC_USAGE, - stream=True, - terminal="unvalidated", - ) - if "responses_terminal" in caps - else None - ), - ( - Case( - name="stream_no_usage_unvalidated", - usage=_BASIC_USAGE, - stream=True, - stream_usage="absent", - terminal="unvalidated", - exact_spend=False, - ) - if "responses_terminal" in caps - else None - ), - ( - Case( - name="prompt_blocked", - usage=_PROMPT_BLOCKED_USAGE, - terminal="prompt_blocked", - response_model_override=True, - ) - if "prompt_blocked" in caps - else None - ), - ( - Case( - name="stream_prompt_blocked", - usage=_PROMPT_BLOCKED_USAGE, - stream=True, - terminal="prompt_blocked", - response_model_override=True, - ) - if "prompt_blocked" in caps - else None - ), - ) - return tuple(case for case in candidates if case is not None) - - -@dataclass(frozen=True, slots=True) -class ExpectedCost: - """The expected bill split the way the spend row's cost_breakdown reports - it: the gross input component (cache reads/writes folded in), the output - component, and the tool-usage component.""" - - input_cost: float - output_cost: float - tool_cost: float - - @property - def total(self) -> float: - return self.input_cost + self.output_cost + self.tool_cost - - -def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: - """Literal arithmetic on the test-map rates over the scripted token counts. - - Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; - output = text*out + reasoning*reasoning + audio_out*audio_out; plus the - billed web-search calls at the medium search-context rate. Above-threshold - swaps every input/output rate to its ``_above_200k_tokens`` variant when - total prompt tokens exceed the threshold; a service tier swaps input/output - to the tier's variants, falling back to the base rate when a variant is - unset -- mirroring _get_token_base_cost in litellm's cost calculator. - """ - rates: Final = model.override_rates if case.response_model_override else model.rates - u: Final = case.usage - prompt_tokens: Final = ( - u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens - + u.cache_write_1h_tokens + u.audio_input_tokens - ) - tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS - in_rate: Final = ( - (rates.input_cost_per_token_above_200k_tokens if tiered else None) - or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.input_cost_per_token - or 0.0 - ) - out_rate: Final = ( - (rates.output_cost_per_token_above_200k_tokens if tiered else None) - or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.output_cost_per_token - or 0.0 - ) - input_cost: Final = ( - u.fresh_input_tokens * in_rate - + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) - + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) - + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) - + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) - ) - output_cost: Final = ( - u.output_tokens * out_rate - + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) - + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) - ) - search: Final = rates.search_context_cost_per_query - tool_cost: Final = case.billed_web_search_calls * ( - search.search_context_size_medium if search and search.search_context_size_medium else 0.0 - ) - return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) - - -def expected_cost(model: FrontierModel, case: Case) -> float: - return expected_breakdown(model, case).total + return tuple(case for case in CASES if case.applies_to(model)) def recount_cost( @@ -738,31 +416,23 @@ def image_input_data_url() -> str: IMAGE_INPUT_DATA_URL: Final = image_input_data_url() -def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: - """(prompt_tokens, completion_tokens) the spend row should carry, per the - wire's normalization: Anthropic folds cache read/write into prompt_tokens, - everyone else reports the totals the wire emitted.""" - u: Final = case.usage - if model.wire in ("anthropic_messages", "bedrock_converse"): - return ( - u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, - u.output_tokens, - ) - if model.wire in ("gemini_generate", "vertex_generate"): - return ( - u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, - u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, - ) - if model.wire == "openai_responses": - return ( - u.fresh_input_tokens + u.cache_read_tokens, - u.output_tokens + u.reasoning_tokens, - ) - return ( - u.fresh_input_tokens - + u.cache_read_tokens - + u.cache_write_5m_tokens - + u.cache_write_1h_tokens - + u.audio_input_tokens, - u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, - ) +class _ExpectedCell(BaseModel): + model_config = ConfigDict(frozen=True) + + spend: float + input_cost: float + output_cost: float + prompt_tokens: int + completion_tokens: int + + +_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, _ExpectedCell]) +EXPECTED: Final[Mapping[str, _ExpectedCell]] = MappingProxyType( + _EXPECTED_ADAPTER.validate_python(json.loads(EXPECTED_PATH.read_text())) + if EXPECTED_PATH.exists() + else {} +) + + +def expected_key(model: FrontierModel, case: Case) -> str: + return f"{model.map_key}|{case.name}" diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json new file mode 100644 index 00000000000..7a92fb2476f --- /dev/null +++ b/tests/e2e/cost_calculation/expected.json @@ -0,0 +1,2004 @@ +{ + "anthropic.claude-sonnet-5-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.03128, + "output_cost": 0.0085, + "prompt_tokens": 150, + "spend": 0.03978 + }, + "anthropic.claude-sonnet-5-v1:0|basic": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.013600000000000001, + "prompt_tokens": 120, + "spend": 0.034 + }, + "anthropic.claude-sonnet-5-v1:0|cache_read": { + "completion_tokens": 30, + "input_cost": 0.01785, + "output_cost": 0.0102, + "prompt_tokens": 150, + "spend": 0.028050000000000002 + }, + "anthropic.claude-sonnet-5-v1:0|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.052700000000000004, + "output_cost": 0.0102, + "prompt_tokens": 150, + "spend": 0.06290000000000001 + }, + "anthropic.claude-sonnet-5-v1:0|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0459, + "output_cost": 0.0102, + "prompt_tokens": 150, + "spend": 0.056100000000000004 + }, + "anthropic.claude-sonnet-5-v1:0|stream": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.013600000000000001, + "prompt_tokens": 120, + "spend": 0.034 + }, + "anthropic.claude-sonnet-5-v1:0|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.013600000000000001, + "output_cost": 0.0085, + "prompt_tokens": 80, + "spend": 0.0221 + }, + "anthropic.claude-sonnet-5-v1:0|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.013600000000000001, + "prompt_tokens": 120, + "spend": 0.034 + }, + "azure/gpt-5.4-mini|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.03424, + "output_cost": 0.02336, + "prompt_tokens": 155, + "spend": 0.0576 + }, + "azure/gpt-5.4-mini|audio": { + "completion_tokens": 45, + "input_cost": 0.04, + "output_cost": 0.0264, + "prompt_tokens": 125, + "spend": 0.0664 + }, + "azure/gpt-5.4-mini|basic": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0168, + "output_cost": 0.009600000000000001, + "prompt_tokens": 150, + "spend": 0.0264 + }, + "azure/gpt-5.4-mini|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.049600000000000005, + "output_cost": 0.009600000000000001, + "prompt_tokens": 150, + "spend": 0.0592 + }, + "azure/gpt-5.4-mini|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0432, + "output_cost": 0.009600000000000001, + "prompt_tokens": 150, + "spend": 0.0528 + }, + "azure/gpt-5.4-mini|reasoning": { + "completion_tokens": 100, + "input_cost": 0.016, + "output_cost": 0.0656, + "prompt_tokens": 100, + "spend": 0.0816 + }, + "azure/gpt-5.4-mini|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0288, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.0448 + }, + "azure/gpt-5.4-mini|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.03264, + "output_cost": 0.01728, + "prompt_tokens": 120, + "spend": 0.049920000000000006 + }, + "azure/gpt-5.4-mini|stream": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0128, + "output_cost": 0.008, + "prompt_tokens": 80, + "spend": 0.0208 + }, + "azure/gpt-5.4-mini|tiered": { + "completion_tokens": 30, + "input_cost": 256.00128, + "output_cost": 0.0432, + "prompt_tokens": 200001, + "spend": 256.04448 + }, + "azure/gpt-5.4-mini|tool_call": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.4-mini|web_search": { + "completion_tokens": 30, + "input_cost": 0.016, + "output_cost": 0.009600000000000001, + "prompt_tokens": 100, + "spend": 0.0456 + }, + "azure/gpt-5.6|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.0321, + "output_cost": 0.0219, + "prompt_tokens": 155, + "spend": 0.05399999999999999 + }, + "azure/gpt-5.6|audio": { + "completion_tokens": 45, + "input_cost": 0.0375, + "output_cost": 0.02475, + "prompt_tokens": 125, + "spend": 0.06225 + }, + "azure/gpt-5.6|basic": { + "completion_tokens": 40, + "input_cost": 0.018, + "output_cost": 0.011999999999999999, + "prompt_tokens": 120, + "spend": 0.03 + }, + "azure/gpt-5.6|cache_read": { + "completion_tokens": 30, + "input_cost": 0.01575, + "output_cost": 0.009, + "prompt_tokens": 150, + "spend": 0.02475 + }, + "azure/gpt-5.6|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0465, + "output_cost": 0.009, + "prompt_tokens": 150, + "spend": 0.0555 + }, + "azure/gpt-5.6|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.040499999999999994, + "output_cost": 0.009, + "prompt_tokens": 150, + "spend": 0.049499999999999995 + }, + "azure/gpt-5.6|reasoning": { + "completion_tokens": 100, + "input_cost": 0.015, + "output_cost": 0.0615, + "prompt_tokens": 100, + "spend": 0.0765 + }, + "azure/gpt-5.6|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.6|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.027, + "output_cost": 0.015, + "prompt_tokens": 120, + "spend": 0.041999999999999996 + }, + "azure/gpt-5.6|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.030600000000000002, + "output_cost": 0.0162, + "prompt_tokens": 120, + "spend": 0.0468 + }, + "azure/gpt-5.6|stream": { + "completion_tokens": 40, + "input_cost": 0.018, + "output_cost": 0.011999999999999999, + "prompt_tokens": 120, + "spend": 0.03 + }, + "azure/gpt-5.6|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.019200000000000002, + "output_cost": 0.0128, + "prompt_tokens": 120, + "spend": 0.032 + }, + "azure/gpt-5.6|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.011999999999999999, + "output_cost": 0.0075, + "prompt_tokens": 80, + "spend": 0.019499999999999997 + }, + "azure/gpt-5.6|tiered": { + "completion_tokens": 30, + "input_cost": 240.00119999999998, + "output_cost": 0.0405, + "prompt_tokens": 200001, + "spend": 240.0417 + }, + "azure/gpt-5.6|tool_call": { + "completion_tokens": 40, + "input_cost": 0.018, + "output_cost": 0.011999999999999999, + "prompt_tokens": 120, + "spend": 0.03 + }, + "azure/gpt-5.6|web_search": { + "completion_tokens": 30, + "input_cost": 0.015, + "output_cost": 0.009, + "prompt_tokens": 100, + "spend": 0.044 + }, + "claude-haiku-4-5|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.012880000000000003, + "output_cost": 0.0035000000000000005, + "prompt_tokens": 150, + "spend": 0.016380000000000002 + }, + "claude-haiku-4-5|all_components_anthropic_stream": { + "completion_tokens": 25, + "input_cost": 0.012880000000000003, + "output_cost": 0.0035000000000000005, + "prompt_tokens": 150, + "spend": 0.016380000000000002 + }, + "claude-haiku-4-5|basic": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-haiku-4-5|cache_read": { + "completion_tokens": 30, + "input_cost": 0.007350000000000001, + "output_cost": 0.004200000000000001, + "prompt_tokens": 150, + "spend": 0.011550000000000001 + }, + "claude-haiku-4-5|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.021700000000000004, + "output_cost": 0.004200000000000001, + "prompt_tokens": 150, + "spend": 0.025900000000000006 + }, + "claude-haiku-4-5|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0189, + "output_cost": 0.004200000000000001, + "prompt_tokens": 150, + "spend": 0.023100000000000002 + }, + "claude-haiku-4-5|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-haiku-4-5|stream": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-haiku-4-5|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-haiku-4-5|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.005600000000000001, + "output_cost": 0.0035000000000000005, + "prompt_tokens": 80, + "spend": 0.0091 + }, + "claude-haiku-4-5|tool_call": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-haiku-4-5|web_search": { + "completion_tokens": 30, + "input_cost": 0.007000000000000001, + "output_cost": 0.004200000000000001, + "prompt_tokens": 100, + "spend": 0.0712 + }, + "claude-opus-5|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.0092, + "output_cost": 0.0025, + "prompt_tokens": 150, + "spend": 0.0117 + }, + "claude-opus-5|all_components_anthropic_stream": { + "completion_tokens": 25, + "input_cost": 0.0092, + "output_cost": 0.0025, + "prompt_tokens": 150, + "spend": 0.0117 + }, + "claude-opus-5|basic": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-opus-5|cache_read": { + "completion_tokens": 30, + "input_cost": 0.00525, + "output_cost": 0.003, + "prompt_tokens": 150, + "spend": 0.00825 + }, + "claude-opus-5|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0155, + "output_cost": 0.003, + "prompt_tokens": 150, + "spend": 0.0185 + }, + "claude-opus-5|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.013500000000000002, + "output_cost": 0.003, + "prompt_tokens": 150, + "spend": 0.0165 + }, + "claude-opus-5|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-opus-5|stream": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-opus-5|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-opus-5|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.004, + "output_cost": 0.0025, + "prompt_tokens": 80, + "spend": 0.006500000000000001 + }, + "claude-opus-5|tool_call": { + "completion_tokens": 40, + "input_cost": 0.006, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.01 + }, + "claude-opus-5|web_search": { + "completion_tokens": 30, + "input_cost": 0.005, + "output_cost": 0.003, + "prompt_tokens": 100, + "spend": 0.068 + }, + "claude-sonnet-5|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.011040000000000001, + "output_cost": 0.0030000000000000005, + "prompt_tokens": 150, + "spend": 0.014040000000000002 + }, + "claude-sonnet-5|all_components_anthropic_stream": { + "completion_tokens": 25, + "input_cost": 0.011040000000000001, + "output_cost": 0.0030000000000000005, + "prompt_tokens": 150, + "spend": 0.014040000000000002 + }, + "claude-sonnet-5|basic": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-sonnet-5|cache_read": { + "completion_tokens": 30, + "input_cost": 0.006300000000000001, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 150, + "spend": 0.0099 + }, + "claude-sonnet-5|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.018600000000000002, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 150, + "spend": 0.0222 + }, + "claude-sonnet-5|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.016200000000000003, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 150, + "spend": 0.0198 + }, + "claude-sonnet-5|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-sonnet-5|stream": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-sonnet-5|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.008400000000000001, + "output_cost": 0.005600000000000001, + "prompt_tokens": 120, + "spend": 0.014000000000000002 + }, + "claude-sonnet-5|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0030000000000000005, + "prompt_tokens": 80, + "spend": 0.007800000000000001 + }, + "claude-sonnet-5|tool_call": { + "completion_tokens": 40, + "input_cost": 0.007200000000000001, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 120, + "spend": 0.012 + }, + "claude-sonnet-5|web_search": { + "completion_tokens": 30, + "input_cost": 0.006000000000000001, + "output_cost": 0.0036000000000000003, + "prompt_tokens": 100, + "spend": 0.0696 + }, + "fireworks_ai/deepseek-v4p1-flash|all_components_fireworks": { + "completion_tokens": 25, + "input_cost": 0.011760000000000001, + "output_cost": 0.007000000000000001, + "prompt_tokens": 120, + "spend": 0.018760000000000002 + }, + "fireworks_ai/deepseek-v4p1-flash|audio": { + "completion_tokens": 45, + "input_cost": 0.030500000000000003, + "output_cost": 0.019950000000000002, + "prompt_tokens": 125, + "spend": 0.05045000000000001 + }, + "fireworks_ai/deepseek-v4p1-flash|basic": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|cache_read": { + "completion_tokens": 30, + "input_cost": 0.014700000000000001, + "output_cost": 0.008400000000000001, + "prompt_tokens": 150, + "spend": 0.023100000000000002 + }, + "fireworks_ai/deepseek-v4p1-flash|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0368, + "output_cost": 0.008400000000000001, + "prompt_tokens": 150, + "spend": 0.045200000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0324, + "output_cost": 0.008400000000000001, + "prompt_tokens": 150, + "spend": 0.0408 + }, + "fireworks_ai/deepseek-v4p1-flash|reasoning": { + "completion_tokens": 100, + "input_cost": 0.014000000000000002, + "output_cost": 0.0469, + "prompt_tokens": 100, + "spend": 0.060899999999999996 + }, + "fireworks_ai/deepseek-v4p1-flash|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/deepseek-v4p1-flash|stream": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/deepseek-v4p1-flash|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.011200000000000002, + "output_cost": 0.007000000000000001, + "prompt_tokens": 80, + "spend": 0.0182 + }, + "fireworks_ai/deepseek-v4p1-flash|tool_call": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/deepseek-v4p1-flash|web_search": { + "completion_tokens": 30, + "input_cost": 0.014000000000000002, + "output_cost": 0.008400000000000001, + "prompt_tokens": 100, + "spend": 0.04240000000000001 + }, + "fireworks_ai/kimi-k3|all_components_fireworks": { + "completion_tokens": 25, + "input_cost": 0.01008, + "output_cost": 0.006000000000000001, + "prompt_tokens": 120, + "spend": 0.01608 + }, + "fireworks_ai/kimi-k3|audio": { + "completion_tokens": 45, + "input_cost": 0.028500000000000004, + "output_cost": 0.01875, + "prompt_tokens": 125, + "spend": 0.04725 + }, + "fireworks_ai/kimi-k3|basic": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/kimi-k3|cache_read": { + "completion_tokens": 30, + "input_cost": 0.012600000000000002, + "output_cost": 0.007200000000000001, + "prompt_tokens": 150, + "spend": 0.0198 + }, + "fireworks_ai/kimi-k3|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.035, + "output_cost": 0.007200000000000001, + "prompt_tokens": 150, + "spend": 0.0422 + }, + "fireworks_ai/kimi-k3|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.030600000000000002, + "output_cost": 0.007200000000000001, + "prompt_tokens": 150, + "spend": 0.0378 + }, + "fireworks_ai/kimi-k3|reasoning": { + "completion_tokens": 100, + "input_cost": 0.012000000000000002, + "output_cost": 0.0457, + "prompt_tokens": 100, + "spend": 0.0577 + }, + "fireworks_ai/kimi-k3|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/kimi-k3|stream": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/kimi-k3|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/kimi-k3|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.009600000000000001, + "output_cost": 0.006000000000000001, + "prompt_tokens": 80, + "spend": 0.015600000000000003 + }, + "fireworks_ai/kimi-k3|tool_call": { + "completion_tokens": 40, + "input_cost": 0.014400000000000001, + "output_cost": 0.009600000000000001, + "prompt_tokens": 120, + "spend": 0.024 + }, + "fireworks_ai/kimi-k3|web_search": { + "completion_tokens": 30, + "input_cost": 0.012000000000000002, + "output_cost": 0.007200000000000001, + "prompt_tokens": 100, + "spend": 0.0392 + }, + "fireworks_ai/qwen3p8-max|all_components_fireworks": { + "completion_tokens": 25, + "input_cost": 0.010920000000000001, + "output_cost": 0.006500000000000001, + "prompt_tokens": 120, + "spend": 0.01742 + }, + "fireworks_ai/qwen3p8-max|audio": { + "completion_tokens": 45, + "input_cost": 0.029500000000000002, + "output_cost": 0.01935, + "prompt_tokens": 125, + "spend": 0.048850000000000005 + }, + "fireworks_ai/qwen3p8-max|basic": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/qwen3p8-max|cache_read": { + "completion_tokens": 30, + "input_cost": 0.01365, + "output_cost": 0.007800000000000001, + "prompt_tokens": 150, + "spend": 0.021450000000000004 + }, + "fireworks_ai/qwen3p8-max|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0359, + "output_cost": 0.007800000000000001, + "prompt_tokens": 150, + "spend": 0.0437 + }, + "fireworks_ai/qwen3p8-max|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0315, + "output_cost": 0.007800000000000001, + "prompt_tokens": 150, + "spend": 0.0393 + }, + "fireworks_ai/qwen3p8-max|reasoning": { + "completion_tokens": 100, + "input_cost": 0.013000000000000001, + "output_cost": 0.0463, + "prompt_tokens": 100, + "spend": 0.059300000000000005 + }, + "fireworks_ai/qwen3p8-max|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/qwen3p8-max|stream": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/qwen3p8-max|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.016800000000000002, + "output_cost": 0.011200000000000002, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "fireworks_ai/qwen3p8-max|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.010400000000000001, + "output_cost": 0.006500000000000001, + "prompt_tokens": 80, + "spend": 0.016900000000000002 + }, + "fireworks_ai/qwen3p8-max|tool_call": { + "completion_tokens": 40, + "input_cost": 0.015600000000000003, + "output_cost": 0.010400000000000001, + "prompt_tokens": 120, + "spend": 0.026000000000000002 + }, + "fireworks_ai/qwen3p8-max|web_search": { + "completion_tokens": 30, + "input_cost": 0.013000000000000001, + "output_cost": 0.007800000000000001, + "prompt_tokens": 100, + "spend": 0.0408 + }, + "gemini-3.1-pro-preview|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.023940000000000003, + "output_cost": 0.030660000000000003, + "prompt_tokens": 125, + "spend": 0.05460000000000001 + }, + "gemini-3.1-pro-preview|audio": { + "completion_tokens": 45, + "input_cost": 0.052500000000000005, + "output_cost": 0.03465, + "prompt_tokens": 125, + "spend": 0.08715 + }, + "gemini-3.1-pro-preview|basic": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.1-pro-preview|cache_read": { + "completion_tokens": 30, + "input_cost": 0.02205, + "output_cost": 0.0126, + "prompt_tokens": 150, + "spend": 0.03465 + }, + "gemini-3.1-pro-preview|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.2, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.2 + }, + "gemini-3.1-pro-preview|reasoning": { + "completion_tokens": 100, + "input_cost": 0.021, + "output_cost": 0.0861, + "prompt_tokens": 100, + "spend": 0.1071 + }, + "gemini-3.1-pro-preview|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.1-pro-preview|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0378, + "output_cost": 0.020999999999999998, + "prompt_tokens": 120, + "spend": 0.0588 + }, + "gemini-3.1-pro-preview|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.04284, + "output_cost": 0.02268, + "prompt_tokens": 120, + "spend": 0.06552 + }, + "gemini-3.1-pro-preview|stream": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.1-pro-preview|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.2, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.2 + }, + "gemini-3.1-pro-preview|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.1-pro-preview|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.016800000000000002, + "output_cost": 0.0105, + "prompt_tokens": 80, + "spend": 0.027300000000000005 + }, + "gemini-3.1-pro-preview|tiered": { + "completion_tokens": 30, + "input_cost": 336.00168, + "output_cost": 0.0567, + "prompt_tokens": 200001, + "spend": 336.05838 + }, + "gemini-3.1-pro-preview|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.1-pro-preview|web_search": { + "completion_tokens": 30, + "input_cost": 0.021, + "output_cost": 0.0126, + "prompt_tokens": 100, + "spend": 0.0936 + }, + "gemini-3.8-flash|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.022799999999999997, + "output_cost": 0.0292, + "prompt_tokens": 125, + "spend": 0.052 + }, + "gemini-3.8-flash|audio": { + "completion_tokens": 45, + "input_cost": 0.05, + "output_cost": 0.033, + "prompt_tokens": 125, + "spend": 0.083 + }, + "gemini-3.8-flash|basic": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.8-flash|cache_read": { + "completion_tokens": 30, + "input_cost": 0.021, + "output_cost": 0.012, + "prompt_tokens": 150, + "spend": 0.033 + }, + "gemini-3.8-flash|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.21000000000000002, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.21000000000000002 + }, + "gemini-3.8-flash|reasoning": { + "completion_tokens": 100, + "input_cost": 0.02, + "output_cost": 0.082, + "prompt_tokens": 100, + "spend": 0.10200000000000001 + }, + "gemini-3.8-flash|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.8-flash|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.036, + "output_cost": 0.02, + "prompt_tokens": 120, + "spend": 0.055999999999999994 + }, + "gemini-3.8-flash|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.0408, + "output_cost": 0.0216, + "prompt_tokens": 120, + "spend": 0.062400000000000004 + }, + "gemini-3.8-flash|stream": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.8-flash|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.21000000000000002, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.21000000000000002 + }, + "gemini-3.8-flash|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0252, + "output_cost": 0.016800000000000002, + "prompt_tokens": 120, + "spend": 0.042 + }, + "gemini-3.8-flash|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.016, + "output_cost": 0.01, + "prompt_tokens": 80, + "spend": 0.026000000000000002 + }, + "gemini-3.8-flash|tiered": { + "completion_tokens": 30, + "input_cost": 320.0016, + "output_cost": 0.054, + "prompt_tokens": 200001, + "spend": 320.05559999999997 + }, + "gemini-3.8-flash|tool_call": { + "completion_tokens": 40, + "input_cost": 0.024, + "output_cost": 0.016, + "prompt_tokens": 120, + "spend": 0.04 + }, + "gemini-3.8-flash|web_search": { + "completion_tokens": 30, + "input_cost": 0.02, + "output_cost": 0.012, + "prompt_tokens": 100, + "spend": 0.092 + }, + "gemini/gemini-3.1-pro-preview|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.010260000000000002, + "output_cost": 0.01314, + "prompt_tokens": 125, + "spend": 0.023400000000000004 + }, + "gemini/gemini-3.1-pro-preview|audio": { + "completion_tokens": 45, + "input_cost": 0.0225, + "output_cost": 0.014849999999999999, + "prompt_tokens": 125, + "spend": 0.037349999999999994 + }, + "gemini/gemini-3.1-pro-preview|basic": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.1-pro-preview|cache_read": { + "completion_tokens": 30, + "input_cost": 0.009450000000000002, + "output_cost": 0.0054, + "prompt_tokens": 150, + "spend": 0.014850000000000002 + }, + "gemini/gemini-3.1-pro-preview|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.08, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.08 + }, + "gemini/gemini-3.1-pro-preview|reasoning": { + "completion_tokens": 100, + "input_cost": 0.009000000000000001, + "output_cost": 0.0369, + "prompt_tokens": 100, + "spend": 0.0459 + }, + "gemini/gemini-3.1-pro-preview|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.1-pro-preview|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0162, + "output_cost": 0.009000000000000001, + "prompt_tokens": 120, + "spend": 0.0252 + }, + "gemini/gemini-3.1-pro-preview|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.01836, + "output_cost": 0.00972, + "prompt_tokens": 120, + "spend": 0.02808 + }, + "gemini/gemini-3.1-pro-preview|stream": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.1-pro-preview|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.08, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.08 + }, + "gemini/gemini-3.1-pro-preview|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.1-pro-preview|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.007200000000000001, + "output_cost": 0.0045000000000000005, + "prompt_tokens": 80, + "spend": 0.011700000000000002 + }, + "gemini/gemini-3.1-pro-preview|tiered": { + "completion_tokens": 30, + "input_cost": 144.00072, + "output_cost": 0.024300000000000002, + "prompt_tokens": 200001, + "spend": 144.02502 + }, + "gemini/gemini-3.1-pro-preview|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.1-pro-preview|web_search": { + "completion_tokens": 30, + "input_cost": 0.009000000000000001, + "output_cost": 0.0054, + "prompt_tokens": 100, + "spend": 0.0744 + }, + "gemini/gemini-3.8-flash|all_components_gemini": { + "completion_tokens": 43, + "input_cost": 0.00912, + "output_cost": 0.01168, + "prompt_tokens": 125, + "spend": 0.0208 + }, + "gemini/gemini-3.8-flash|audio": { + "completion_tokens": 45, + "input_cost": 0.02, + "output_cost": 0.0132, + "prompt_tokens": 125, + "spend": 0.0332 + }, + "gemini/gemini-3.8-flash|basic": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.8-flash|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0084, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 150, + "spend": 0.0132 + }, + "gemini/gemini-3.8-flash|prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.09000000000000001, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.09000000000000001 + }, + "gemini/gemini-3.8-flash|reasoning": { + "completion_tokens": 100, + "input_cost": 0.008, + "output_cost": 0.0328, + "prompt_tokens": 100, + "spend": 0.0408 + }, + "gemini/gemini-3.8-flash|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.8-flash|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0144, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.0224 + }, + "gemini/gemini-3.8-flash|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.01632, + "output_cost": 0.00864, + "prompt_tokens": 120, + "spend": 0.024960000000000003 + }, + "gemini/gemini-3.8-flash|stream": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.8-flash|stream_prompt_blocked": { + "completion_tokens": 0, + "input_cost": 0.09000000000000001, + "output_cost": 0.0, + "prompt_tokens": 1000, + "spend": 0.09000000000000001 + }, + "gemini/gemini-3.8-flash|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0108, + "output_cost": 0.007200000000000001, + "prompt_tokens": 120, + "spend": 0.018000000000000002 + }, + "gemini/gemini-3.8-flash|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0064, + "output_cost": 0.004, + "prompt_tokens": 80, + "spend": 0.0104 + }, + "gemini/gemini-3.8-flash|tiered": { + "completion_tokens": 30, + "input_cost": 128.00064, + "output_cost": 0.0216, + "prompt_tokens": 200001, + "spend": 128.02224 + }, + "gemini/gemini-3.8-flash|tool_call": { + "completion_tokens": 40, + "input_cost": 0.009600000000000001, + "output_cost": 0.0064, + "prompt_tokens": 120, + "spend": 0.016 + }, + "gemini/gemini-3.8-flash|web_search": { + "completion_tokens": 30, + "input_cost": 0.008, + "output_cost": 0.0048000000000000004, + "prompt_tokens": 100, + "spend": 0.0728 + }, + "gpt-5.3-codex|all_components_responses": { + "completion_tokens": 40, + "input_cost": 0.00252, + "output_cost": 0.0037500000000000007, + "prompt_tokens": 120, + "spend": 0.006270000000000001 + }, + "gpt-5.3-codex|basic": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0031500000000000005, + "output_cost": 0.0018000000000000002, + "prompt_tokens": 150, + "spend": 0.00495 + }, + "gpt-5.3-codex|reasoning": { + "completion_tokens": 100, + "input_cost": 0.0030000000000000005, + "output_cost": 0.0123, + "prompt_tokens": 100, + "spend": 0.015300000000000001 + }, + "gpt-5.3-codex|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.3-codex|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0054, + "output_cost": 0.003, + "prompt_tokens": 120, + "spend": 0.008400000000000001 + }, + "gpt-5.3-codex|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00612, + "output_cost": 0.00324, + "prompt_tokens": 120, + "spend": 0.00936 + }, + "gpt-5.3-codex|stream": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|stream_incomplete": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.3-codex|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0015000000000000002, + "prompt_tokens": 80, + "spend": 0.0039000000000000007 + }, + "gpt-5.3-codex|stream_unvalidated": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|tiered": { + "completion_tokens": 30, + "input_cost": 48.000240000000005, + "output_cost": 0.0081, + "prompt_tokens": 200001, + "spend": 48.008340000000004 + }, + "gpt-5.3-codex|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.3-codex|web_search": { + "completion_tokens": 30, + "input_cost": 0.0030000000000000005, + "output_cost": 0.0018000000000000002, + "prompt_tokens": 100, + "spend": 0.0648 + }, + "gpt-5.4-mini|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.00856, + "output_cost": 0.00584, + "prompt_tokens": 155, + "spend": 0.0144 + }, + "gpt-5.4-mini|audio": { + "completion_tokens": 45, + "input_cost": 0.01, + "output_cost": 0.0066, + "prompt_tokens": 125, + "spend": 0.0166 + }, + "gpt-5.4-mini|basic": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.4-mini|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0042, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 150, + "spend": 0.0066 + }, + "gpt-5.4-mini|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.012400000000000001, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 150, + "spend": 0.0148 + }, + "gpt-5.4-mini|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0108, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 150, + "spend": 0.0132 + }, + "gpt-5.4-mini|reasoning": { + "completion_tokens": 100, + "input_cost": 0.004, + "output_cost": 0.0164, + "prompt_tokens": 100, + "spend": 0.0204 + }, + "gpt-5.4-mini|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.4-mini|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0072, + "output_cost": 0.004, + "prompt_tokens": 120, + "spend": 0.0112 + }, + "gpt-5.4-mini|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00816, + "output_cost": 0.00432, + "prompt_tokens": 120, + "spend": 0.012480000000000002 + }, + "gpt-5.4-mini|stream": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.4-mini|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.4-mini|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0032, + "output_cost": 0.002, + "prompt_tokens": 80, + "spend": 0.0052 + }, + "gpt-5.4-mini|tiered": { + "completion_tokens": 30, + "input_cost": 64.00032, + "output_cost": 0.0108, + "prompt_tokens": 200001, + "spend": 64.01112 + }, + "gpt-5.4-mini|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.4-mini|web_search": { + "completion_tokens": 30, + "input_cost": 0.004, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 100, + "spend": 0.0264 + }, + "gpt-5.5-pro|all_components_responses": { + "completion_tokens": 40, + "input_cost": 0.00168, + "output_cost": 0.0025, + "prompt_tokens": 120, + "spend": 0.00418 + }, + "gpt-5.5-pro|basic": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0021, + "output_cost": 0.0012000000000000001, + "prompt_tokens": 150, + "spend": 0.0033 + }, + "gpt-5.5-pro|reasoning": { + "completion_tokens": 100, + "input_cost": 0.002, + "output_cost": 0.0082, + "prompt_tokens": 100, + "spend": 0.0102 + }, + "gpt-5.5-pro|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.5-pro|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0036, + "output_cost": 0.002, + "prompt_tokens": 120, + "spend": 0.0056 + }, + "gpt-5.5-pro|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00408, + "output_cost": 0.00216, + "prompt_tokens": 120, + "spend": 0.006240000000000001 + }, + "gpt-5.5-pro|stream": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|stream_incomplete": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0036000000000000003, + "output_cost": 0.0024000000000000002, + "prompt_tokens": 120, + "spend": 0.006 + }, + "gpt-5.5-pro|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0016, + "output_cost": 0.001, + "prompt_tokens": 80, + "spend": 0.0026 + }, + "gpt-5.5-pro|stream_unvalidated": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|tiered": { + "completion_tokens": 30, + "input_cost": 32.00016, + "output_cost": 0.0054, + "prompt_tokens": 200001, + "spend": 32.00556 + }, + "gpt-5.5-pro|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0024000000000000002, + "output_cost": 0.0016, + "prompt_tokens": 120, + "spend": 0.004 + }, + "gpt-5.5-pro|web_search": { + "completion_tokens": 30, + "input_cost": 0.002, + "output_cost": 0.0012000000000000001, + "prompt_tokens": 100, + "spend": 0.06319999999999999 + }, + "gpt-5.6|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.00214, + "output_cost": 0.00146, + "prompt_tokens": 155, + "spend": 0.0036 + }, + "gpt-5.6|audio": { + "completion_tokens": 45, + "input_cost": 0.0025, + "output_cost": 0.00165, + "prompt_tokens": 125, + "spend": 0.00415 + }, + "gpt-5.6|basic": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.6|cache_read": { + "completion_tokens": 30, + "input_cost": 0.00105, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 150, + "spend": 0.00165 + }, + "gpt-5.6|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0031000000000000003, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 150, + "spend": 0.0037 + }, + "gpt-5.6|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.0027, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 150, + "spend": 0.0033 + }, + "gpt-5.6|reasoning": { + "completion_tokens": 100, + "input_cost": 0.001, + "output_cost": 0.0041, + "prompt_tokens": 100, + "spend": 0.0051 + }, + "gpt-5.6|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.6|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.0018, + "output_cost": 0.001, + "prompt_tokens": 120, + "spend": 0.0028 + }, + "gpt-5.6|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.00204, + "output_cost": 0.00108, + "prompt_tokens": 120, + "spend": 0.0031200000000000004 + }, + "gpt-5.6|stream": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.6|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0048000000000000004, + "output_cost": 0.0032, + "prompt_tokens": 120, + "spend": 0.008 + }, + "gpt-5.6|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0008, + "output_cost": 0.0005, + "prompt_tokens": 80, + "spend": 0.0013 + }, + "gpt-5.6|tiered": { + "completion_tokens": 30, + "input_cost": 16.00008, + "output_cost": 0.0027, + "prompt_tokens": 200001, + "spend": 16.00278 + }, + "gpt-5.6|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0012000000000000001, + "output_cost": 0.0008, + "prompt_tokens": 120, + "spend": 0.002 + }, + "gpt-5.6|web_search": { + "completion_tokens": 30, + "input_cost": 0.001, + "output_cost": 0.0006000000000000001, + "prompt_tokens": 100, + "spend": 0.0216 + }, + "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.020900000000000002, + "output_cost": 0.0095, + "prompt_tokens": 150, + "spend": 0.030400000000000003 + }, + "meta.llama4-maverick-17b-instruct-v1:0|basic": { + "completion_tokens": 40, + "input_cost": 0.0228, + "output_cost": 0.015200000000000002, + "prompt_tokens": 120, + "spend": 0.038000000000000006 + }, + "meta.llama4-maverick-17b-instruct-v1:0|stream": { + "completion_tokens": 40, + "input_cost": 0.0228, + "output_cost": 0.015200000000000002, + "prompt_tokens": 120, + "spend": 0.038000000000000006 + }, + "meta.llama4-maverick-17b-instruct-v1:0|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.015200000000000002, + "output_cost": 0.0095, + "prompt_tokens": 80, + "spend": 0.0247 + }, + "meta.llama4-maverick-17b-instruct-v1:0|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0228, + "output_cost": 0.015200000000000002, + "prompt_tokens": 120, + "spend": 0.038000000000000006 + }, + "together_ai/moonshotai/Kimi-K3|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.0214, + "output_cost": 0.0146, + "prompt_tokens": 155, + "spend": 0.036 + }, + "together_ai/moonshotai/Kimi-K3|audio": { + "completion_tokens": 45, + "input_cost": 0.025, + "output_cost": 0.0165, + "prompt_tokens": 125, + "spend": 0.0415 + }, + "together_ai/moonshotai/Kimi-K3|basic": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/moonshotai/Kimi-K3|cache_read": { + "completion_tokens": 30, + "input_cost": 0.0105, + "output_cost": 0.006, + "prompt_tokens": 150, + "spend": 0.0165 + }, + "together_ai/moonshotai/Kimi-K3|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.031, + "output_cost": 0.006, + "prompt_tokens": 150, + "spend": 0.037 + }, + "together_ai/moonshotai/Kimi-K3|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.027000000000000003, + "output_cost": 0.006, + "prompt_tokens": 150, + "spend": 0.033 + }, + "together_ai/moonshotai/Kimi-K3|reasoning": { + "completion_tokens": 100, + "input_cost": 0.01, + "output_cost": 0.041, + "prompt_tokens": 100, + "spend": 0.051000000000000004 + }, + "together_ai/moonshotai/Kimi-K3|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/moonshotai/Kimi-K3|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.018000000000000002, + "output_cost": 0.01, + "prompt_tokens": 120, + "spend": 0.028000000000000004 + }, + "together_ai/moonshotai/Kimi-K3|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.0204, + "output_cost": 0.0108, + "prompt_tokens": 120, + "spend": 0.031200000000000002 + }, + "together_ai/moonshotai/Kimi-K3|stream": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/moonshotai/Kimi-K3|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/moonshotai/Kimi-K3|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.008, + "output_cost": 0.005, + "prompt_tokens": 80, + "spend": 0.013000000000000001 + }, + "together_ai/moonshotai/Kimi-K3|tiered": { + "completion_tokens": 30, + "input_cost": 160.0008, + "output_cost": 0.027000000000000003, + "prompt_tokens": 200001, + "spend": 160.02779999999998 + }, + "together_ai/moonshotai/Kimi-K3|tool_call": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/moonshotai/Kimi-K3|web_search": { + "completion_tokens": 30, + "input_cost": 0.01, + "output_cost": 0.006, + "prompt_tokens": 100, + "spend": 0.036000000000000004 + }, + "together_ai/zai-org/GLM-5.3|all_components_chat": { + "completion_tokens": 43, + "input_cost": 0.023540000000000002, + "output_cost": 0.01606, + "prompt_tokens": 155, + "spend": 0.0396 + }, + "together_ai/zai-org/GLM-5.3|audio": { + "completion_tokens": 45, + "input_cost": 0.027500000000000004, + "output_cost": 0.01815, + "prompt_tokens": 125, + "spend": 0.04565 + }, + "together_ai/zai-org/GLM-5.3|basic": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/zai-org/GLM-5.3|cache_read": { + "completion_tokens": 30, + "input_cost": 0.011550000000000001, + "output_cost": 0.0066, + "prompt_tokens": 150, + "spend": 0.01815 + }, + "together_ai/zai-org/GLM-5.3|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.034100000000000005, + "output_cost": 0.0066, + "prompt_tokens": 150, + "spend": 0.04070000000000001 + }, + "together_ai/zai-org/GLM-5.3|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.029699999999999997, + "output_cost": 0.0066, + "prompt_tokens": 150, + "spend": 0.0363 + }, + "together_ai/zai-org/GLM-5.3|reasoning": { + "completion_tokens": 100, + "input_cost": 0.011000000000000001, + "output_cost": 0.0451, + "prompt_tokens": 100, + "spend": 0.056100000000000004 + }, + "together_ai/zai-org/GLM-5.3|response_model_override": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/zai-org/GLM-5.3|service_tier_flex": { + "completion_tokens": 40, + "input_cost": 0.019799999999999998, + "output_cost": 0.011000000000000001, + "prompt_tokens": 120, + "spend": 0.0308 + }, + "together_ai/zai-org/GLM-5.3|service_tier_priority": { + "completion_tokens": 40, + "input_cost": 0.022439999999999998, + "output_cost": 0.01188, + "prompt_tokens": 120, + "spend": 0.034319999999999996 + }, + "together_ai/zai-org/GLM-5.3|stream": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/zai-org/GLM-5.3|stream_response_model_override": { + "completion_tokens": 40, + "input_cost": 0.012, + "output_cost": 0.008, + "prompt_tokens": 120, + "spend": 0.02 + }, + "together_ai/zai-org/GLM-5.3|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.0088, + "output_cost": 0.0055000000000000005, + "prompt_tokens": 80, + "spend": 0.0143 + }, + "together_ai/zai-org/GLM-5.3|tiered": { + "completion_tokens": 30, + "input_cost": 176.00088, + "output_cost": 0.0297, + "prompt_tokens": 200001, + "spend": 176.03058 + }, + "together_ai/zai-org/GLM-5.3|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0132, + "output_cost": 0.0088, + "prompt_tokens": 120, + "spend": 0.022 + }, + "together_ai/zai-org/GLM-5.3|web_search": { + "completion_tokens": 30, + "input_cost": 0.011000000000000001, + "output_cost": 0.0066, + "prompt_tokens": 100, + "spend": 0.0376 + }, + "us.anthropic.claude-opus-5-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.033120000000000004, + "output_cost": 0.009000000000000001, + "prompt_tokens": 150, + "spend": 0.042120000000000005 + }, + "us.anthropic.claude-opus-5-v1:0|basic": { + "completion_tokens": 40, + "input_cost": 0.0216, + "output_cost": 0.014400000000000001, + "prompt_tokens": 120, + "spend": 0.036000000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|cache_read": { + "completion_tokens": 30, + "input_cost": 0.018900000000000004, + "output_cost": 0.0108, + "prompt_tokens": 150, + "spend": 0.029700000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|cache_write_1h": { + "completion_tokens": 30, + "input_cost": 0.0558, + "output_cost": 0.0108, + "prompt_tokens": 150, + "spend": 0.0666 + }, + "us.anthropic.claude-opus-5-v1:0|cache_write_5m": { + "completion_tokens": 30, + "input_cost": 0.048600000000000004, + "output_cost": 0.0108, + "prompt_tokens": 150, + "spend": 0.05940000000000001 + }, + "us.anthropic.claude-opus-5-v1:0|stream": { + "completion_tokens": 40, + "input_cost": 0.0216, + "output_cost": 0.014400000000000001, + "prompt_tokens": 120, + "spend": 0.036000000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|stream_tool_call": { + "completion_tokens": 25, + "input_cost": 0.014400000000000001, + "output_cost": 0.009000000000000001, + "prompt_tokens": 80, + "spend": 0.023400000000000004 + }, + "us.anthropic.claude-opus-5-v1:0|tool_call": { + "completion_tokens": 40, + "input_cost": 0.0216, + "output_cost": 0.014400000000000001, + "prompt_tokens": 120, + "spend": 0.036000000000000004 + } +} diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py new file mode 100644 index 00000000000..de979f272fe --- /dev/null +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -0,0 +1,189 @@ +"""Golden generator for the cost suite. Run: + + uv run python tests/e2e/cost_calculation/generate_expected.py + +Loads the derived matrix (models x applicable cases), computes the golden for +each exact-spend cell from the rate arithmetic, and writes ``expected.json`` +with sorted keys. Default behaviour adds missing cells and drops stale cells +but never overwrites an existing cell's values (a reviewed golden is +authoritative); ``--rewrite`` recomputes everything. Prints added/removed/kept +counts. +""" + +from __future__ import annotations + +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from cost_matrix import ( # noqa: E402 # path bootstrap before package-local imports + EXPECTED_PATH, + FRONTIER_MODELS, + TIER_THRESHOLD_TOKENS, + Case, + CostMapEntry, + FrontierModel, + cases_for, + expected_key, +) + +# Wires whose response surface reports a real web-search call count; the +# chat-completions wires only expose url_citation annotations, so their billed +# count floors to one. +_EXACT_WEB_SEARCH_WIRES: Final = frozenset( + {"openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"} +) + + +def billed_web_search_calls(model: FrontierModel, case: Case) -> int: + if case.usage.web_search_calls == 0: + return 0 + return case.usage.web_search_calls if model.wire in _EXACT_WEB_SEARCH_WIRES else 1 + + +@dataclass(frozen=True, slots=True) +class ExpectedCost: + """The expected bill split the way the spend row's cost_breakdown reports + it: the gross input component (cache reads/writes folded in), the output + component, and the tool-usage component.""" + + input_cost: float + output_cost: float + tool_cost: float + + @property + def total(self) -> float: + return self.input_cost + self.output_cost + self.tool_cost + + +def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: + """Literal arithmetic on the test-map rates over the scripted token counts. + + Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; + output = text*out + reasoning*reasoning + audio_out*audio_out; plus the + billed web-search calls at the medium search-context rate. Above-threshold + swaps every input/output rate to its ``_above_200k_tokens`` variant when + total prompt tokens exceed the threshold; a service tier swaps input/output + to the tier's variants, falling back to the base rate when a variant is + unset -- mirroring _get_token_base_cost in litellm's cost calculator. + """ + rates: Final[CostMapEntry] = model.override_rates if case.response_model_override else model.rates + u: Final = case.usage + prompt_tokens: Final = ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + u.audio_input_tokens + ) + tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS + in_rate: Final = ( + (rates.input_cost_per_token_above_200k_tokens if tiered else None) + or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.input_cost_per_token + or 0.0 + ) + out_rate: Final = ( + (rates.output_cost_per_token_above_200k_tokens if tiered else None) + or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) + or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) + or rates.output_cost_per_token + or 0.0 + ) + # The biller charges cache writes at the input rate when the entry carries + # no cache_creation rate (cost_calculator.py:2452), and at the 5m write + # rate when the 1h variant is unset; cache reads bill only at their own + # rate (zero when the entry lacks one). + write_5m_rate: Final = rates.cache_creation_input_token_cost or in_rate + input_cost: Final = ( + u.fresh_input_tokens * in_rate + + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) + + u.cache_write_5m_tokens * write_5m_rate + + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or write_5m_rate) + + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) + ) + output_cost: Final = ( + u.output_tokens * out_rate + + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) + + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) + ) + search: Final = rates.search_context_cost_per_query + tool_cost: Final = billed_web_search_calls(model, case) * ( + search.search_context_size_medium if search and search.search_context_size_medium else 0.0 + ) + return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) + + +def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: + """(prompt_tokens, completion_tokens) the spend row should carry, per the + wire's normalization: Anthropic folds cache read/write into prompt_tokens, + everyone else reports the totals the wire emitted.""" + u: Final = case.usage + if model.wire in ("anthropic_messages", "bedrock_converse"): + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, + u.output_tokens, + ) + if model.wire in ("gemini_generate", "vertex_generate"): + return ( + u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) + if model.wire == "openai_responses": + return ( + u.fresh_input_tokens + u.cache_read_tokens, + u.output_tokens + u.reasoning_tokens, + ) + return ( + u.fresh_input_tokens + + u.cache_read_tokens + + u.cache_write_5m_tokens + + u.cache_write_1h_tokens + + u.audio_input_tokens, + u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, + ) + + +def _proposed() -> dict[str, dict[str, object]]: + return { + expected_key(model, case): ( + lambda breakdown, tokens: { + "spend": breakdown.total, + "input_cost": breakdown.input_cost, + "output_cost": breakdown.output_cost, + "prompt_tokens": tokens[0], + "completion_tokens": tokens[1], + } + )(expected_breakdown(model, case), expected_token_columns(model, case)) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + + +def main() -> None: + rewrite: Final = "--rewrite" in sys.argv[1:] + proposed: Final = _proposed() + existing: Final = ( + json.loads(EXPECTED_PATH.read_text()) if EXPECTED_PATH.exists() else {} + ) + merged: Final = { + key: (proposed[key] if rewrite or key not in existing else existing[key]) + for key in sorted(proposed) + } + added: Final = sum(1 for key in proposed if key not in existing) + removed: Final = sum(1 for key in existing if key not in proposed) + kept: Final = sum(1 for key in proposed if key in existing and not rewrite) + rewritten: Final = sum(1 for key in proposed if key in existing and rewrite) + EXPECTED_PATH.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n") + print( + f"expected.json: {added} added, {removed} removed, {kept} kept, " + f"{rewritten} rewritten ({len(merged)} cells)" + ) + + +if __name__ == "__main__": + main() diff --git a/tests/e2e/cost_calculation/test_matrix_data.py b/tests/e2e/cost_calculation/test_matrix_data.py new file mode 100644 index 00000000000..fdbb6ddd293 --- /dev/null +++ b/tests/e2e/cost_calculation/test_matrix_data.py @@ -0,0 +1,64 @@ +"""Freshness checks for the cost suite's data files; markerless, so it runs on +any pytest invocation of the folder without the stack. expected.json is the +oracle: these tests check its key set against the derived matrix, never its +values (the generator proposes, the file decides).""" + +from __future__ import annotations + +from typing import Final + +import pytest + +from cost_matrix import ( + _CASES_FILE, + _COST_MAP, + CASES, + EXPECTED, + FRONTIER_MODELS, + CostMapEntry, + cases_for, + expected_key, +) + + +def test_expected_keys_match_derived_exact_cells() -> None: + derived: Final = { + expected_key(model, case) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + golden: Final = set(EXPECTED) + if derived != golden: + missing: Final = sorted(derived - golden) + stale: Final = sorted(golden - derived) + pytest.fail( + "expected.json is out of sync with the derived matrix; run " + "uv run python tests/e2e/cost_calculation/generate_expected.py " + f"(missing: {missing}; stale: {stale})" + ) + + +def test_deployments_reference_existing_map_keys() -> None: + unknown: Final = sorted( + spec.map_key for spec in _CASES_FILE.deployments if spec.map_key not in _COST_MAP + ) + assert not unknown, f"deployments entries name map keys absent from cost_map.json: {unknown}" + + +def test_requires_rates_are_cost_map_fields() -> None: + fields: Final = set(CostMapEntry.model_fields) + unknown: Final = sorted( + {field for case in CASES for field in case.requires_rates} - fields + ) + assert not unknown, f"requires_rates names that are not CostMapEntry fields: {unknown}" + + +def test_no_two_entries_share_input_rate() -> None: + rates: Final = [ + entry.input_cost_per_token for entry in _COST_MAP.values() + ] + assert len(rates) == len(set(rates)), ( + "two cost_map entries share input_cost_per_token; the suite relies on " + "distinct rates so a wrong-model bill can never coincidentally match" + ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 0b4f3e1fd37..7cd128ad6fb 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -1,7 +1,7 @@ -"""Token-pricing e2e: every (frontier model, pricing-component case) cell runs a -scripted-usage call through a deployment registered on the cost-map proxy, and -the spend row plus response-cost header must equal literal arithmetic on the -test map's rates. +"""Token-pricing e2e: every (map entry, case) cell derived from cost_map.json x +cases.json runs a scripted-usage call through a deployment registered on the +cost-map proxy, and the spend row plus response-cost header must equal the +reviewed golden in expected.json verbatim -- no rate arithmetic lives here. Nothing here touches a real provider or the bundled cost map: the proxy's upstream is the scripted-provider sidecar and its entire cost map is @@ -15,13 +15,13 @@ from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( + EXPECTED, FRONTIER_MODELS, IMAGE_INPUT_DATA_URL, Case, FrontierModel, cases_for, - expected_cost, - expected_token_columns, + expected_key, recount_cost, ) from e2e_config import unique_marker @@ -110,17 +110,6 @@ class TestTokenPricing: ) assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - expected: Final = expected_cost(model, case) - if case.exact_spend and not case.stream: - # Streamed responses commit headers before the bill is computed, so - # the x-litellm-response-cost header is asserted only on non-stream - # calls. - assert response.response_cost is not None and cost_rows.approx_equal( - response.response_cost, expected - ), ( - f"x-litellm-response-cost {response.response_cost} != expected {expected}" - ) - row: Final = cost_rows.poll_cost_row_where( client.proxy, scoped_key, @@ -149,16 +138,39 @@ class TestTokenPricing: cost_rows.assert_total_is_sum_of_components(row) return - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected), ( - f"{model.map_key}/{case.name}: spend {row.spend} != expected {expected} " + golden: Final = EXPECTED[expected_key(model, case)] + + if not case.stream: + # Streamed responses commit headers before the bill is computed, so + # the x-litellm-response-cost header is asserted only on non-stream + # calls. + assert response.response_cost is not None and cost_rows.approx_equal( + response.response_cost, golden.spend + ), ( + f"x-litellm-response-cost {response.response_cost} != golden {golden.spend}" + ) + + assert row.spend is not None and cost_rows.approx_equal(row.spend, golden.spend), ( + f"{model.map_key}/{case.name}: spend {row.spend} != golden {golden.spend} " f"(breakdown {row.breakdown.model_dump()})" ) - - prompt_tokens, completion_tokens = expected_token_columns(model, case) - assert row.prompt_tokens == prompt_tokens, ( - f"prompt_tokens {row.prompt_tokens} != {prompt_tokens}" + breakdown: Final = row.breakdown + assert breakdown.input_cost is not None and cost_rows.approx_equal( + breakdown.input_cost, golden.input_cost + ), ( + f"{model.map_key}/{case.name}: gross input_cost {breakdown.input_cost} " + f"!= golden {golden.input_cost}; cached/written tokens billed at the input rate" ) - assert row.completion_tokens == completion_tokens, ( - f"completion_tokens {row.completion_tokens} != {completion_tokens}" + assert breakdown.output_cost is not None and cost_rows.approx_equal( + breakdown.output_cost, golden.output_cost + ), ( + f"{model.map_key}/{case.name}: output_cost {breakdown.output_cost} " + f"!= golden {golden.output_cost}" + ) + assert row.prompt_tokens == golden.prompt_tokens, ( + f"prompt_tokens {row.prompt_tokens} != {golden.prompt_tokens}" + ) + assert row.completion_tokens == golden.completion_tokens, ( + f"completion_tokens {row.completion_tokens} != {golden.completion_tokens}" ) cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/cost_calculation/test_wire_formats_e2e.py b/tests/e2e/cost_calculation/test_wire_formats_e2e.py deleted file mode 100644 index a36bb1a8662..00000000000 --- a/tests/e2e/cost_calculation/test_wire_formats_e2e.py +++ /dev/null @@ -1,368 +0,0 @@ -"""Wire-format e2e: one scripted upstream per provider wire, answering with a -usage payload where every token kind the wire can report is nonzero. The spend -row's gross input cost must equal fresh tokens at the input rate plus each cache -and audio component at its own rate -- proving the wire's usage shape landed the -cached tokens inside the total (OpenAI/Gemini) or as separate fields -(Anthropic), and that the biller subtracted them before billing fresh tokens. - -Also covers the Responses API wire (an openai/gpt-5.5-pro deployment bridged by -the proxy to POST /responses) and a streamed Anthropic-messages case. -""" - -from __future__ import annotations - -import pytest -from collections.abc import Mapping -from types import MappingProxyType -from typing import Final - -from conftest import CostCalcClient, cost_rows, register_scenario_deployment -from cost_matrix import ( - FRONTIER_MODELS, - Case, - FrontierModel, - expected_breakdown, - expected_token_columns, -) -from e2e_config import unique_marker -from lifecycle import ResourceManager -from models import ChatBody, ChatMessage, ChatStreamOptions, ChatTool, ChatToolFunction -from scripted_provider import ScriptedUsage - -pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark - -_MODELS: Final[Mapping[str, FrontierModel]] = MappingProxyType( - {model.map_key: model for model in FRONTIER_MODELS} -) - -# One scripted usage per wire, every reportable token kind nonzero. -_WIRE_USAGE: Final[Mapping[str, tuple[str, ScriptedUsage]]] = MappingProxyType({ - "openai_chat": ( - "gpt-5.6", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "openai_responses": ( - "gpt-5.5-pro", - ScriptedUsage( - fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25, reasoning_tokens=15 - ), - ), - "anthropic_messages": ( - "claude-sonnet-5", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - ), - ), - "gemini_generate": ( - "gemini/gemini-3.8-flash", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "together_chat": ( - "together_ai/moonshotai/Kimi-K3", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "fireworks_chat": ( - "fireworks_ai/kimi-k3", - ScriptedUsage(fresh_input_tokens=80, cache_read_tokens=40, output_tokens=25), - ), - "azure_chat": ( - "azure/gpt-5.6", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), - "bedrock_converse": ( - "anthropic.claude-sonnet-5-v1:0", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - cache_write_5m_tokens=20, - cache_write_1h_tokens=10, - output_tokens=25, - ), - ), - "vertex_generate": ( - "gemini-3.8-flash", - ScriptedUsage( - fresh_input_tokens=80, - cache_read_tokens=40, - output_tokens=25, - reasoning_tokens=15, - audio_input_tokens=5, - audio_output_tokens=3, - ), - ), -}) - -_SHAPE_USAGE: Final = ScriptedUsage(fresh_input_tokens=80, output_tokens=25) - -# Renderer-level shapes the pricing matrix gates per cap, pinned here once per -# wire so the sidecar emits prove they survive the proxy end to end. -_SHAPES: Final[tuple[tuple[str, str, Case], ...]] = ( - *( - ( - f"tool_call_{'stream' if stream else 'sync'}", - wire, - Case(name="tool_call", usage=_SHAPE_USAGE, stream=stream, tool_call=True), - ) - for wire in _WIRE_USAGE - for stream in (False, True) - ), - ( - "responses_incomplete", - "openai_responses", - Case(name="stream_no_usage_incomplete", usage=_SHAPE_USAGE, stream=True, terminal="incomplete"), - ), - ( - "responses_unvalidated", - "openai_responses", - Case(name="stream_unvalidated", usage=_SHAPE_USAGE, stream=True, terminal="unvalidated"), - ), - ( - "gemini_prompt_blocked", - "gemini_generate", - Case( - name="prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "gemini_prompt_blocked_stream", - "gemini_generate", - Case( - name="stream_prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - stream=True, - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "vertex_prompt_blocked", - "vertex_generate", - Case( - name="prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "vertex_prompt_blocked_stream", - "vertex_generate", - Case( - name="stream_prompt_blocked", - usage=ScriptedUsage(fresh_input_tokens=1000, output_tokens=0), - stream=True, - terminal="prompt_blocked", - response_model_override=True, - ), - ), - ( - "azure_served_model_override", - "azure_chat", - Case( - name="response_model_override", - usage=_SHAPE_USAGE, - response_model_override=True, - ), - ), -) - - -def _shape_id(entry: tuple[str, str, Case]) -> str: - return entry[0] - - -class TestWireFormats: - @pytest.mark.parametrize("wire", tuple(_WIRE_USAGE)) - @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") - def test_wire_usage_shape_bills_each_component( - self, - client: CostCalcClient, - resources: ResourceManager, - scoped_key: str, - wire: str, - ) -> None: - map_key, usage = _WIRE_USAGE[wire] - model: Final = _MODELS[map_key] - case: Final = Case(name="basic", usage=usage) - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted wire call"),), - ), - ) - assert response.ok, f"{wire}: proxy returned {response.status_code}: {response.body[:400]}" - - expected: Final = expected_breakdown(model, case) - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, f"{wire}: no spend row landed" - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( - f"{wire}: spend {row.spend} != expected {expected.total} " - f"(breakdown {row.breakdown.model_dump()})" - ) - breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and cost_rows.approx_equal( - breakdown.input_cost, expected.input_cost - ), ( - f"{wire}: gross input_cost {breakdown.input_cost} != expected {expected.input_cost}; " - "cached/written tokens billed at the input rate" - ) - assert breakdown.output_cost is not None and cost_rows.approx_equal( - breakdown.output_cost, expected.output_cost - ), f"{wire}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" - - prompt_tokens, completion_tokens = expected_token_columns(model, case) - assert row.prompt_tokens == prompt_tokens, ( - f"{wire}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" - ) - assert row.completion_tokens == completion_tokens, ( - f"{wire}: completion_tokens {row.completion_tokens} != {completion_tokens}" - ) - cost_rows.assert_total_is_sum_of_components(row) - - @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") - def test_anthropic_streamed_usage_bills_each_component( - self, client: CostCalcClient, resources: ResourceManager, scoped_key: str - ) -> None: - map_key, usage = _WIRE_USAGE["anthropic_messages"] - model: Final = _MODELS[map_key] - case: Final = Case(name="stream", usage=usage, stream=True) - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted anthropic stream"),), - stream=True, - stream_options=ChatStreamOptions(include_usage=True), - ), - stream=True, - ) - assert response.ok, f"anthropic stream: proxy returned {response.status_code}: {response.body[:400]}" - assert response.stream_done, "anthropic stream did not reach its terminal event" - assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - - expected: Final = expected_breakdown(model, case) - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, "anthropic stream: no spend row landed" - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( - f"anthropic stream: spend {row.spend} != expected {expected.total} " - f"(breakdown {row.breakdown.model_dump()})" - ) - cost_rows.assert_total_is_sum_of_components(row) - - @pytest.mark.parametrize("shape_wire_case", _SHAPES, ids=_shape_id) - @pytest.mark.covers("quota_management.spend_tracking.scripted_wire.logs_cost") - def test_response_shape_bills_reported_usage( - self, - client: CostCalcClient, - resources: ResourceManager, - scoped_key: str, - shape_wire_case: tuple[str, str, Case], - ) -> None: - shape, wire, case = shape_wire_case - map_key, _usage = _WIRE_USAGE[wire] - model: Final = _MODELS[map_key] - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=model_name, - messages=(ChatMessage(role="user", content=f"{marker} scripted {shape}"),), - stream=case.stream, - stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, - tools=( - ( - ChatTool( - function=ChatToolFunction( - name="get_weather", - parameters={"type": "object", "properties": {"city": {"type": "string"}}}, - ) - ), - ) - if case.tool_call - else None - ), - ), - stream=case.stream, - ) - assert response.ok, f"{shape}: proxy returned {response.status_code}: {response.body[:400]}" - if case.stream: - assert response.stream_done, f"{shape}: stream did not reach its terminal event" - assert response.stream_error is None, f"{shape}: stream error: {response.stream_error}" - - expected: Final = expected_breakdown(model, case) - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, f"{shape}: no spend row landed" - assert row.spend is not None and cost_rows.approx_equal(row.spend, expected.total), ( - f"{shape}: spend {row.spend} != expected {expected.total} " - f"(breakdown {row.breakdown.model_dump()})" - ) - prompt_tokens, completion_tokens = expected_token_columns(model, case) - assert row.prompt_tokens == prompt_tokens, ( - f"{shape}: prompt_tokens {row.prompt_tokens} != {prompt_tokens}" - ) - assert row.completion_tokens == completion_tokens, ( - f"{shape}: completion_tokens {row.completion_tokens} != {completion_tokens}" - ) - cost_rows.assert_total_is_sum_of_components(row) From 522a7f569283b9a3bc0fed2a66f1c7545f30b12b Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 00:51:43 +0000 Subject: [PATCH 11/30] test(e2e): gate all_components cases by rates and tidy cost matrix names Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cases.json | 26 ++++++++ tests/e2e/cost_calculation/cost_matrix.py | 31 +++++---- tests/e2e/cost_calculation/expected.json | 7 --- .../e2e/cost_calculation/generate_expected.py | 63 ++++++++++--------- .../e2e/cost_calculation/test_matrix_data.py | 11 ++-- 5 files changed, 79 insertions(+), 59 deletions(-) diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index e898557ea35..3dc4fc4d99c 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -180,11 +180,20 @@ "audio_input_tokens": 5, "audio_output_tokens": 3 }, + "requires_rates": [ + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr", + "output_cost_per_reasoning_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token" + ], "wires": ["openai_chat", "azure_chat", "together_chat"] }, { "name": "all_components_fireworks", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, + "requires_rates": ["cache_read_input_token_cost"], "wires": ["fireworks_chat"] }, { @@ -196,6 +205,11 @@ "cache_write_1h_tokens": 10, "output_tokens": 25 }, + "requires_rates": [ + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr" + ], "wires": ["anthropic_messages", "bedrock_converse"] }, { @@ -208,6 +222,11 @@ "output_tokens": 25 }, "stream": true, + "requires_rates": [ + "cache_read_input_token_cost", + "cache_creation_input_token_cost", + "cache_creation_input_token_cost_above_1hr" + ], "wires": ["anthropic_messages"] }, { @@ -220,11 +239,18 @@ "audio_input_tokens": 5, "audio_output_tokens": 3 }, + "requires_rates": [ + "cache_read_input_token_cost", + "output_cost_per_reasoning_token", + "input_cost_per_audio_token", + "output_cost_per_audio_token" + ], "wires": ["gemini_generate", "vertex_generate"] }, { "name": "all_components_responses", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, + "requires_rates": ["cache_read_input_token_cost", "output_cost_per_reasoning_token"], "wires": ["openai_responses"] } ] diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index b03c851d208..a8f60b79ae7 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -27,7 +27,6 @@ from types import MappingProxyType from typing import Final, Literal from pydantic import BaseModel, ConfigDict, TypeAdapter - from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" @@ -69,9 +68,9 @@ class CostMapEntry(BaseModel): web_search_billing_unit: str | None = None -_COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) -_COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( - _COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) +COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) +COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( + COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) ) TIER_THRESHOLD_TOKENS: Final = 200_000 @@ -146,10 +145,10 @@ class _CasesFile(BaseModel): cases: tuple[Case, ...] = () -_CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) -CASES: Final[tuple[Case, ...]] = _CASES_FILE.cases +CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) +CASES: Final[tuple[Case, ...]] = CASES_FILE.cases _DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( - {spec.map_key: spec for spec in _CASES_FILE.deployments} + {spec.map_key: spec for spec in CASES_FILE.deployments} ) @@ -221,13 +220,13 @@ class FrontierModel: @property def rates(self) -> CostMapEntry: - return _COST_MAP[self.map_key] + return COST_MAP[self.map_key] @property def override_rates(self) -> CostMapEntry: if self.base_model is not None or self.override_map_key is None: return self.rates - return _COST_MAP[self.override_map_key] + return COST_MAP[self.override_map_key] @property def provider_model(self) -> str: @@ -262,13 +261,13 @@ def _litellm_model_for(map_key: str, wiring: _ProviderWiring) -> str: def _frontier() -> tuple[FrontierModel, ...]: groups: Final[Mapping[tuple[str, str], tuple[str, ...]]] = MappingProxyType( { - pair: tuple(sorted(k for k, e in _COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) - for pair in {(e.litellm_provider, e.mode) for e in _COST_MAP.values()} + pair: tuple(sorted(k for k, e in COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) + for pair in {(e.litellm_provider, e.mode) for e in COST_MAP.values()} } ) models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple - for map_key in sorted(_COST_MAP): - entry: Final = _COST_MAP[map_key] + for map_key in sorted(COST_MAP): + entry: Final = COST_MAP[map_key] pair: Final = (entry.litellm_provider, entry.mode) wiring: Final = _PROVIDER_WIRING.get(pair) if wiring is None: @@ -416,7 +415,7 @@ def image_input_data_url() -> str: IMAGE_INPUT_DATA_URL: Final = image_input_data_url() -class _ExpectedCell(BaseModel): +class ExpectedCell(BaseModel): model_config = ConfigDict(frozen=True) spend: float @@ -426,8 +425,8 @@ class _ExpectedCell(BaseModel): completion_tokens: int -_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, _ExpectedCell]) -EXPECTED: Final[Mapping[str, _ExpectedCell]] = MappingProxyType( +_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, ExpectedCell]) +EXPECTED: Final[Mapping[str, ExpectedCell]] = MappingProxyType( _EXPECTED_ADAPTER.validate_python(json.loads(EXPECTED_PATH.read_text())) if EXPECTED_PATH.exists() else {} diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json index 7a92fb2476f..3b18e9ed9f4 100644 --- a/tests/e2e/cost_calculation/expected.json +++ b/tests/e2e/cost_calculation/expected.json @@ -1686,13 +1686,6 @@ "prompt_tokens": 100, "spend": 0.0216 }, - "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.020900000000000002, - "output_cost": 0.0095, - "prompt_tokens": 150, - "spend": 0.030400000000000003 - }, "meta.llama4-maverick-17b-instruct-v1:0|basic": { "completion_tokens": 40, "input_cost": 0.0228, diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index de979f272fe..c093ecbe0ea 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -14,8 +14,10 @@ from __future__ import annotations import json import sys +from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Final sys.path.insert(0, str(Path(__file__).resolve().parent)) @@ -27,6 +29,7 @@ from cost_matrix import ( # noqa: E402 # path bootstrap before package-local i TIER_THRESHOLD_TOKENS, Case, CostMapEntry, + ExpectedCell, FrontierModel, cases_for, expected_key, @@ -93,16 +96,11 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: or rates.output_cost_per_token or 0.0 ) - # The biller charges cache writes at the input rate when the entry carries - # no cache_creation rate (cost_calculator.py:2452), and at the 5m write - # rate when the 1h variant is unset; cache reads bill only at their own - # rate (zero when the entry lacks one). - write_5m_rate: Final = rates.cache_creation_input_token_cost or in_rate input_cost: Final = ( u.fresh_input_tokens * in_rate + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) - + u.cache_write_5m_tokens * write_5m_rate - + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or write_5m_rate) + + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) + + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) ) output_cost: Final = ( @@ -147,39 +145,46 @@ def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: ) -def _proposed() -> dict[str, dict[str, object]]: - return { - expected_key(model, case): ( - lambda breakdown, tokens: { - "spend": breakdown.total, - "input_cost": breakdown.input_cost, - "output_cost": breakdown.output_cost, - "prompt_tokens": tokens[0], - "completion_tokens": tokens[1], - } - )(expected_breakdown(model, case), expected_token_columns(model, case)) - for model in FRONTIER_MODELS - for case in cases_for(model) - if case.exact_spend - } +def _cell(model: FrontierModel, case: Case) -> ExpectedCell: + breakdown: Final = expected_breakdown(model, case) + prompt_tokens, completion_tokens = expected_token_columns(model, case) + return ExpectedCell( + spend=breakdown.total, + input_cost=breakdown.input_cost, + output_cost=breakdown.output_cost, + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + ) + + +def _proposed() -> Mapping[str, ExpectedCell]: + return MappingProxyType( + { + expected_key(model, case): _cell(model, case) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + ) def main() -> None: rewrite: Final = "--rewrite" in sys.argv[1:] proposed: Final = _proposed() + proposed_values: Final = {key: cell.model_dump() for key, cell in proposed.items()} existing: Final = ( json.loads(EXPECTED_PATH.read_text()) if EXPECTED_PATH.exists() else {} ) merged: Final = { - key: (proposed[key] if rewrite or key not in existing else existing[key]) - for key in sorted(proposed) + key: (proposed_values[key] if rewrite or key not in existing else existing[key]) + for key in sorted(proposed_values) } - added: Final = sum(1 for key in proposed if key not in existing) - removed: Final = sum(1 for key in existing if key not in proposed) - kept: Final = sum(1 for key in proposed if key in existing and not rewrite) - rewritten: Final = sum(1 for key in proposed if key in existing and rewrite) + added: Final = sum(1 for key in proposed_values if key not in existing) + removed: Final = sum(1 for key in existing if key not in proposed_values) + kept: Final = sum(1 for key in proposed_values if key in existing and not rewrite) + rewritten: Final = sum(1 for key in proposed_values if key in existing and rewrite) EXPECTED_PATH.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n") - print( + print( # noqa: T201 # CLI summary is the tool output f"expected.json: {added} added, {removed} removed, {kept} kept, " f"{rewritten} rewritten ({len(merged)} cells)" ) diff --git a/tests/e2e/cost_calculation/test_matrix_data.py b/tests/e2e/cost_calculation/test_matrix_data.py index fdbb6ddd293..8340257939c 100644 --- a/tests/e2e/cost_calculation/test_matrix_data.py +++ b/tests/e2e/cost_calculation/test_matrix_data.py @@ -8,11 +8,10 @@ from __future__ import annotations from typing import Final import pytest - from cost_matrix import ( - _CASES_FILE, - _COST_MAP, CASES, + CASES_FILE, + COST_MAP, EXPECTED, FRONTIER_MODELS, CostMapEntry, @@ -41,7 +40,7 @@ def test_expected_keys_match_derived_exact_cells() -> None: def test_deployments_reference_existing_map_keys() -> None: unknown: Final = sorted( - spec.map_key for spec in _CASES_FILE.deployments if spec.map_key not in _COST_MAP + spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP ) assert not unknown, f"deployments entries name map keys absent from cost_map.json: {unknown}" @@ -55,9 +54,7 @@ def test_requires_rates_are_cost_map_fields() -> None: def test_no_two_entries_share_input_rate() -> None: - rates: Final = [ - entry.input_cost_per_token for entry in _COST_MAP.values() - ] + rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) assert len(rates) == len(set(rates)), ( "two cost_map entries share input_cost_per_token; the suite relies on " "distinct rates so a wrong-model bill can never coincidentally match" From e1c9ae5ae45e3b66041a25a6ae6ca9c7633944b7 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 00:56:39 +0000 Subject: [PATCH 12/30] test(e2e): drop needless sys.path bootstrap from golden generator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/generate_expected.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index c093ecbe0ea..e243e477839 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -16,14 +16,10 @@ import json import sys from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path from types import MappingProxyType from typing import Final -sys.path.insert(0, str(Path(__file__).resolve().parent)) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - -from cost_matrix import ( # noqa: E402 # path bootstrap before package-local imports +from cost_matrix import ( EXPECTED_PATH, FRONTIER_MODELS, TIER_THRESHOLD_TOKENS, From 3e11c986766ed7a32ead704e5284fbfeaf889c6b Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:02:41 +0000 Subject: [PATCH 13/30] test(e2e): satisfy pyright in cost matrix derivation and golden generator Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cost_matrix.py | 14 +++++++------- tests/e2e/cost_calculation/generate_expected.py | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index a8f60b79ae7..35344a099be 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -267,23 +267,23 @@ def _frontier() -> tuple[FrontierModel, ...]: ) models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple for map_key in sorted(COST_MAP): - entry: Final = COST_MAP[map_key] - pair: Final = (entry.litellm_provider, entry.mode) - wiring: Final = _PROVIDER_WIRING.get(pair) + entry = COST_MAP[map_key] + pair = (entry.litellm_provider, entry.mode) + wiring = _PROVIDER_WIRING.get(pair) if wiring is None: raise ValueError( f"cost_map entry {map_key} has no wiring for " f"(litellm_provider={pair[0]}, mode={pair[1]}); add a " f"_ProviderWiring row in cost_matrix.py" ) - siblings: Final = groups[pair] - override_key: Final = ( + siblings = groups[pair] + override_key = ( siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None ) - override_litellm: Final = ( + override_litellm = ( _litellm_model_for(override_key, wiring) if override_key is not None else None ) - deployment: Final = _DEPLOYMENTS.get(map_key) + deployment = _DEPLOYMENTS.get(map_key) models.append( FrontierModel( model_name=f"cc-{map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index e243e477839..f5514092c88 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -19,6 +19,8 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final +from pydantic import TypeAdapter + from cost_matrix import ( EXPECTED_PATH, FRONTIER_MODELS, @@ -168,11 +170,19 @@ def main() -> None: rewrite: Final = "--rewrite" in sys.argv[1:] proposed: Final = _proposed() proposed_values: Final = {key: cell.model_dump() for key, cell in proposed.items()} - existing: Final = ( - json.loads(EXPECTED_PATH.read_text()) if EXPECTED_PATH.exists() else {} + existing: Final[Mapping[str, ExpectedCell]] = ( + TypeAdapter(dict[str, ExpectedCell]).validate_python( + json.loads(EXPECTED_PATH.read_text()) + ) + if EXPECTED_PATH.exists() + else {} ) merged: Final = { - key: (proposed_values[key] if rewrite or key not in existing else existing[key]) + key: ( + proposed_values[key] + if rewrite or key not in existing + else existing[key].model_dump() + ) for key in sorted(proposed_values) } added: Final = sum(1 for key in proposed_values if key not in existing) From fc0cce553a631e912e7892893bde188e9716b415 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:13:36 +0000 Subject: [PATCH 14/30] test(e2e): derive cache rates from first principles and ungate all_components cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cases.json | 17 +---------------- tests/e2e/cost_calculation/expected.json | 7 +++++++ tests/e2e/cost_calculation/generate_expected.py | 17 ++++++++++++++--- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index 3dc4fc4d99c..e01ac97e9ff 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -181,9 +181,6 @@ "audio_output_tokens": 3 }, "requires_rates": [ - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr", "output_cost_per_reasoning_token", "input_cost_per_audio_token", "output_cost_per_audio_token" @@ -193,7 +190,6 @@ { "name": "all_components_fireworks", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, - "requires_rates": ["cache_read_input_token_cost"], "wires": ["fireworks_chat"] }, { @@ -205,11 +201,6 @@ "cache_write_1h_tokens": 10, "output_tokens": 25 }, - "requires_rates": [ - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr" - ], "wires": ["anthropic_messages", "bedrock_converse"] }, { @@ -222,11 +213,6 @@ "output_tokens": 25 }, "stream": true, - "requires_rates": [ - "cache_read_input_token_cost", - "cache_creation_input_token_cost", - "cache_creation_input_token_cost_above_1hr" - ], "wires": ["anthropic_messages"] }, { @@ -240,7 +226,6 @@ "audio_output_tokens": 3 }, "requires_rates": [ - "cache_read_input_token_cost", "output_cost_per_reasoning_token", "input_cost_per_audio_token", "output_cost_per_audio_token" @@ -250,7 +235,7 @@ { "name": "all_components_responses", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, - "requires_rates": ["cache_read_input_token_cost", "output_cost_per_reasoning_token"], + "requires_rates": ["output_cost_per_reasoning_token"], "wires": ["openai_responses"] } ] diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json index 3b18e9ed9f4..caea2c3c764 100644 --- a/tests/e2e/cost_calculation/expected.json +++ b/tests/e2e/cost_calculation/expected.json @@ -1686,6 +1686,13 @@ "prompt_tokens": 100, "spend": 0.0216 }, + "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { + "completion_tokens": 25, + "input_cost": 0.0285, + "output_cost": 0.0095, + "prompt_tokens": 150, + "spend": 0.038 + }, "meta.llama4-maverick-17b-instruct-v1:0|basic": { "completion_tokens": 40, "input_cost": 0.0228, diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index f5514092c88..a6eabcc7286 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -94,11 +94,22 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: or rates.output_cost_per_token or 0.0 ) + write_rate: Final = ( + rates.cache_creation_input_token_cost + if rates.cache_creation_input_token_cost is not None + else in_rate + ) input_cost: Final = ( u.fresh_input_tokens * in_rate - + u.cache_read_tokens * (rates.cache_read_input_token_cost or 0.0) - + u.cache_write_5m_tokens * (rates.cache_creation_input_token_cost or 0.0) - + u.cache_write_1h_tokens * (rates.cache_creation_input_token_cost_above_1hr or 0.0) + + u.cache_read_tokens + * (rates.cache_read_input_token_cost if rates.cache_read_input_token_cost is not None else in_rate) + + u.cache_write_5m_tokens * write_rate + + u.cache_write_1h_tokens + * ( + rates.cache_creation_input_token_cost_above_1hr + if rates.cache_creation_input_token_cost_above_1hr is not None + else write_rate + ) + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) ) output_cost: Final = ( From 072b32baf2097e5421672956ce34809106f592aa Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:18:02 +0000 Subject: [PATCH 15/30] test(e2e): derive goldens from first-principles rate selection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/cases.json | 10 ++- tests/e2e/cost_calculation/expected.json | 18 ++-- .../e2e/cost_calculation/generate_expected.py | 86 +++++++++---------- 3 files changed, 61 insertions(+), 53 deletions(-) diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index e01ac97e9ff..49eebc85231 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -62,7 +62,15 @@ "name": "web_search", "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 3}, "requires_rates": ["search_context_cost_per_query"], - "requires_caps": ["web_search"] + "requires_caps": ["web_search"], + "wires": ["openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"] + }, + { + "name": "web_search_single", + "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 1}, + "requires_rates": ["search_context_cost_per_query"], + "requires_caps": ["web_search"], + "wires": ["openai_chat", "together_chat", "fireworks_chat", "azure_chat"] }, { "name": "stream", diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json index caea2c3c764..984b670a82c 100644 --- a/tests/e2e/cost_calculation/expected.json +++ b/tests/e2e/cost_calculation/expected.json @@ -160,7 +160,7 @@ "prompt_tokens": 120, "spend": 0.032 }, - "azure/gpt-5.4-mini|web_search": { + "azure/gpt-5.4-mini|web_search_single": { "completion_tokens": 30, "input_cost": 0.016, "output_cost": 0.009600000000000001, @@ -272,7 +272,7 @@ "prompt_tokens": 120, "spend": 0.03 }, - "azure/gpt-5.6|web_search": { + "azure/gpt-5.6|web_search_single": { "completion_tokens": 30, "input_cost": 0.015, "output_cost": 0.009, @@ -615,7 +615,7 @@ "prompt_tokens": 120, "spend": 0.028000000000000004 }, - "fireworks_ai/deepseek-v4p1-flash|web_search": { + "fireworks_ai/deepseek-v4p1-flash|web_search_single": { "completion_tokens": 30, "input_cost": 0.014000000000000002, "output_cost": 0.008400000000000001, @@ -706,7 +706,7 @@ "prompt_tokens": 120, "spend": 0.024 }, - "fireworks_ai/kimi-k3|web_search": { + "fireworks_ai/kimi-k3|web_search_single": { "completion_tokens": 30, "input_cost": 0.012000000000000002, "output_cost": 0.007200000000000001, @@ -797,7 +797,7 @@ "prompt_tokens": 120, "spend": 0.026000000000000002 }, - "fireworks_ai/qwen3p8-max|web_search": { + "fireworks_ai/qwen3p8-max|web_search_single": { "completion_tokens": 30, "input_cost": 0.013000000000000001, "output_cost": 0.007800000000000001, @@ -1462,7 +1462,7 @@ "prompt_tokens": 120, "spend": 0.008 }, - "gpt-5.4-mini|web_search": { + "gpt-5.4-mini|web_search_single": { "completion_tokens": 30, "input_cost": 0.004, "output_cost": 0.0024000000000000002, @@ -1679,7 +1679,7 @@ "prompt_tokens": 120, "spend": 0.002 }, - "gpt-5.6|web_search": { + "gpt-5.6|web_search_single": { "completion_tokens": 30, "input_cost": 0.001, "output_cost": 0.0006000000000000001, @@ -1826,7 +1826,7 @@ "prompt_tokens": 120, "spend": 0.02 }, - "together_ai/moonshotai/Kimi-K3|web_search": { + "together_ai/moonshotai/Kimi-K3|web_search_single": { "completion_tokens": 30, "input_cost": 0.01, "output_cost": 0.006, @@ -1938,7 +1938,7 @@ "prompt_tokens": 120, "spend": 0.022 }, - "together_ai/zai-org/GLM-5.3|web_search": { + "together_ai/zai-org/GLM-5.3|web_search_single": { "completion_tokens": 30, "input_cost": 0.011000000000000001, "output_cost": 0.0066, diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py index a6eabcc7286..64abdb14c99 100644 --- a/tests/e2e/cost_calculation/generate_expected.py +++ b/tests/e2e/cost_calculation/generate_expected.py @@ -19,8 +19,6 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final -from pydantic import TypeAdapter - from cost_matrix import ( EXPECTED_PATH, FRONTIER_MODELS, @@ -32,19 +30,11 @@ from cost_matrix import ( cases_for, expected_key, ) - -# Wires whose response surface reports a real web-search call count; the -# chat-completions wires only expose url_citation annotations, so their billed -# count floors to one. -_EXACT_WEB_SEARCH_WIRES: Final = frozenset( - {"openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"} -) +from pydantic import TypeAdapter -def billed_web_search_calls(model: FrontierModel, case: Case) -> int: - if case.usage.web_search_calls == 0: - return 0 - return case.usage.web_search_calls if model.wire in _EXACT_WEB_SEARCH_WIRES else 1 +def _first_present(*rates: float | None) -> float | None: + return next((rate for rate in rates if rate is not None), None) @dataclass(frozen=True, slots=True) @@ -67,11 +57,14 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; output = text*out + reasoning*reasoning + audio_out*audio_out; plus the - billed web-search calls at the medium search-context rate. Above-threshold - swaps every input/output rate to its ``_above_200k_tokens`` variant when - total prompt tokens exceed the threshold; a service tier swaps input/output - to the tier's variants, falling back to the base rate when a variant is - unset -- mirroring _get_token_base_cost in litellm's cost calculator. + billed web-search calls at the medium search-context rate. Every billed + token is a token the provider charged for: a component whose entry has no + dedicated rate bills at the ordinary input or output rate, and a present + rate (including an explicit 0.0) is authoritative. When the total prompt + tokens exceed the threshold, input/output rates come from the + ``_above_200k_tokens`` variants; a service tier takes its ``_priority`` or + ``_flex`` variant when the entry carries one, and otherwise bills at the + base rate. """ rates: Final[CostMapEntry] = model.override_rates if case.response_model_override else model.rates u: Final = case.usage @@ -81,46 +74,53 @@ def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: ) tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS in_rate: Final = ( - (rates.input_cost_per_token_above_200k_tokens if tiered else None) - or (rates.input_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.input_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.input_cost_per_token + _first_present( + rates.input_cost_per_token_above_200k_tokens if tiered else None, + rates.input_cost_per_token_priority if case.service_tier == "priority" else None, + rates.input_cost_per_token_flex if case.service_tier == "flex" else None, + rates.input_cost_per_token, + ) or 0.0 ) out_rate: Final = ( - (rates.output_cost_per_token_above_200k_tokens if tiered else None) - or (rates.output_cost_per_token_priority if case.service_tier == "priority" else None) - or (rates.output_cost_per_token_flex if case.service_tier == "flex" else None) - or rates.output_cost_per_token + _first_present( + rates.output_cost_per_token_above_200k_tokens if tiered else None, + rates.output_cost_per_token_priority if case.service_tier == "priority" else None, + rates.output_cost_per_token_flex if case.service_tier == "flex" else None, + rates.output_cost_per_token, + ) or 0.0 ) - write_rate: Final = ( - rates.cache_creation_input_token_cost - if rates.cache_creation_input_token_cost is not None - else in_rate + read_rate: Final = _first_present(rates.cache_read_input_token_cost, in_rate) or 0.0 + write_rate: Final = _first_present(rates.cache_creation_input_token_cost, in_rate) or 0.0 + write_1h_rate: Final = ( + _first_present(rates.cache_creation_input_token_cost_above_1hr, write_rate) or 0.0 ) + audio_in_rate: Final = _first_present(rates.input_cost_per_audio_token, in_rate) or 0.0 + reasoning_rate: Final = _first_present(rates.output_cost_per_reasoning_token, out_rate) or 0.0 + audio_out_rate: Final = _first_present(rates.output_cost_per_audio_token, out_rate) or 0.0 input_cost: Final = ( u.fresh_input_tokens * in_rate - + u.cache_read_tokens - * (rates.cache_read_input_token_cost if rates.cache_read_input_token_cost is not None else in_rate) + + u.cache_read_tokens * read_rate + u.cache_write_5m_tokens * write_rate - + u.cache_write_1h_tokens - * ( - rates.cache_creation_input_token_cost_above_1hr - if rates.cache_creation_input_token_cost_above_1hr is not None - else write_rate - ) - + u.audio_input_tokens * (rates.input_cost_per_audio_token or 0.0) + + u.cache_write_1h_tokens * write_1h_rate + + u.audio_input_tokens * audio_in_rate ) output_cost: Final = ( u.output_tokens * out_rate - + u.reasoning_tokens * (rates.output_cost_per_reasoning_token or out_rate) - + u.audio_output_tokens * (rates.output_cost_per_audio_token or out_rate) + + u.reasoning_tokens * reasoning_rate + + u.audio_output_tokens * audio_out_rate ) search: Final = rates.search_context_cost_per_query - tool_cost: Final = billed_web_search_calls(model, case) * ( - search.search_context_size_medium if search and search.search_context_size_medium else 0.0 + medium_rate: Final = ( + search.search_context_size_medium if search is not None else None ) + if u.web_search_calls and medium_rate is None: + raise ValueError( + f"{model.map_key}: case {case.name} bills {u.web_search_calls} web-search " + "calls but the entry has no search_context_cost_per_query medium rate" + ) + tool_cost: Final = u.web_search_calls * (medium_rate if medium_rate is not None else 0.0) return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) From 1de633ac36644c5774cf629a793b6716a98b7580 Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:22:01 +0000 Subject: [PATCH 16/30] test(e2e): move matrix data freshness checks to collection time Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/cost_calculation/cost_matrix.py | 48 +++++++++++++++ .../e2e/cost_calculation/test_matrix_data.py | 61 ------------------- .../test_token_pricing_e2e.py | 4 ++ 4 files changed, 53 insertions(+), 62 deletions(-) delete mode 100644 tests/e2e/cost_calculation/test_matrix_data.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 707d35b4aa6..f89b3203622 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`; `cost_matrix.matrix_data_errors()` runs at collection time so a stale key set fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 35344a099be..68f3186809d 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -435,3 +435,51 @@ EXPECTED: Final[Mapping[str, ExpectedCell]] = MappingProxyType( def expected_key(model: FrontierModel, case: Case) -> str: return f"{model.map_key}|{case.name}" + + +def matrix_data_errors() -> tuple[str, ...]: + """Freshness findings for the data files, as human-readable strings. + + Called at collection time by the e2e suite; also usable from + generate_expected.py's context without importing pytest. + """ + derived: Final = { + expected_key(model, case) + for model in FRONTIER_MODELS + for case in cases_for(model) + if case.exact_spend + } + golden: Final = set(EXPECTED) + unknown_deployments: Final = sorted( + spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP + ) + unknown_rates: Final = sorted( + {field for case in CASES for field in case.requires_rates} - set(CostMapEntry.model_fields) + ) + input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) + findings: Final = ( + ( + "expected.json is out of sync with the derived matrix; run " + "uv run python tests/e2e/cost_calculation/generate_expected.py " + f"(missing: {sorted(derived - golden)}; stale: {sorted(golden - derived)})" + ) + if derived != golden + else None, + ( + f"deployments entries name map keys absent from cost_map.json: {unknown_deployments}" + if unknown_deployments + else None + ), + ( + f"requires_rates names that are not CostMapEntry fields: {unknown_rates}" + if unknown_rates + else None + ), + ( + "two cost_map entries share input_cost_per_token; the suite relies on " + "distinct rates so a wrong-model bill can never coincidentally match" + if len(input_rates) != len(set(input_rates)) + else None + ), + ) + return tuple(finding for finding in findings if finding is not None) diff --git a/tests/e2e/cost_calculation/test_matrix_data.py b/tests/e2e/cost_calculation/test_matrix_data.py deleted file mode 100644 index 8340257939c..00000000000 --- a/tests/e2e/cost_calculation/test_matrix_data.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Freshness checks for the cost suite's data files; markerless, so it runs on -any pytest invocation of the folder without the stack. expected.json is the -oracle: these tests check its key set against the derived matrix, never its -values (the generator proposes, the file decides).""" - -from __future__ import annotations - -from typing import Final - -import pytest -from cost_matrix import ( - CASES, - CASES_FILE, - COST_MAP, - EXPECTED, - FRONTIER_MODELS, - CostMapEntry, - cases_for, - expected_key, -) - - -def test_expected_keys_match_derived_exact_cells() -> None: - derived: Final = { - expected_key(model, case) - for model in FRONTIER_MODELS - for case in cases_for(model) - if case.exact_spend - } - golden: Final = set(EXPECTED) - if derived != golden: - missing: Final = sorted(derived - golden) - stale: Final = sorted(golden - derived) - pytest.fail( - "expected.json is out of sync with the derived matrix; run " - "uv run python tests/e2e/cost_calculation/generate_expected.py " - f"(missing: {missing}; stale: {stale})" - ) - - -def test_deployments_reference_existing_map_keys() -> None: - unknown: Final = sorted( - spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP - ) - assert not unknown, f"deployments entries name map keys absent from cost_map.json: {unknown}" - - -def test_requires_rates_are_cost_map_fields() -> None: - fields: Final = set(CostMapEntry.model_fields) - unknown: Final = sorted( - {field for case in CASES for field in case.requires_rates} - fields - ) - assert not unknown, f"requires_rates names that are not CostMapEntry fields: {unknown}" - - -def test_no_two_entries_share_input_rate() -> None: - rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) - assert len(rates) == len(set(rates)), ( - "two cost_map entries share input_cost_per_token; the suite relies on " - "distinct rates so a wrong-model bill can never coincidentally match" - ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 7cd128ad6fb..346a55aa22d 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -22,6 +22,7 @@ from cost_matrix import ( FrontierModel, cases_for, expected_key, + matrix_data_errors, recount_cost, ) from e2e_config import unique_marker @@ -39,6 +40,9 @@ from models import ( pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark +if _data_errors := matrix_data_errors(): + raise ValueError("\n".join(_data_errors)) + _MATRIX: Final[tuple[tuple[FrontierModel, Case], ...]] = tuple( (model, case) for model in FRONTIER_MODELS for case in cases_for(model) ) From ef1f306a7dc0777276166859b3fa4d2e6272cefb Mon Sep 17 00:00:00 2001 From: kerry Date: Thu, 17 Sep 2026 01:53:52 +0000 Subject: [PATCH 17/30] test(e2e): emit gemini stream usage only on the final chunk Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/cost_calculation/scripted_provider.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 982132ed8df..90d95441e5c 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -750,10 +750,8 @@ def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, objec def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: emit_usage: Final = scenario.stream_usage == "final_chunk" - first: Final = ( - _jobj(*((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata")) - if scenario.stream_usage == "absent" - else _gemini_body(scenario, requested_model) + first: Final = _jobj( + *((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata") ) return _sse( ( From aac1456e07ef0bce7dd2ec23aaff66b96e7e565c Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 18 Sep 2026 06:11:00 +0000 Subject: [PATCH 18/30] refactor(e2e): inline literal expected costs into cases.json Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/cost_calculation/cases.json | 510 ++++- tests/e2e/cost_calculation/conftest.py | 5 +- tests/e2e/cost_calculation/cost_matrix.py | 175 +- tests/e2e/cost_calculation/expected.json | 2004 ----------------- .../e2e/cost_calculation/generate_expected.py | 211 -- .../test_token_pricing_e2e.py | 7 +- 7 files changed, 477 insertions(+), 2437 deletions(-) delete mode 100644 tests/e2e/cost_calculation/expected.json delete mode 100644 tests/e2e/cost_calculation/generate_expected.py diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index f89b3203622..a3e5696ef9d 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` and `expected.json` (regenerate with `generate_expected.py`; `cost_matrix.matrix_data_errors()` runs at collection time so a stale key set fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` (each exact-spend case carries a literal `expected` cell per map key; `cost_matrix.matrix_data_errors()` runs at collection time so a key absent from the cost map fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index 49eebc85231..cda7bc6e67a 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -1,81 +1,254 @@ { "deployments": [ - { - "map_key": "azure/gpt-5.4-mini", - "litellm_model": "azure/cc-pinned-deployment", - "base_model": "azure/gpt-5.4-mini" - } + {"map_key": "azure/gpt-5.4-mini", "litellm_model": "azure/cc-pinned-deployment", "base_model": "azure/gpt-5.4-mini"} ], "cases": [ { "name": "basic", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40} + "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.034, "input_cost": 0.0204, "output_cost": 0.0136, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.03, "input_cost": 0.018, "output_cost": 0.012, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-haiku-4-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-opus-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-sonnet-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/kimi-k3": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/qwen3p8-max": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, + "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0228, "output_cost": 0.0152, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.036, "input_cost": 0.0216, "output_cost": 0.0144, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "cache_read", "usage": {"fresh_input_tokens": 100, "cache_read_tokens": 50, "output_tokens": 30}, - "requires_rates": ["cache_read_input_token_cost"], - "requires_caps": ["cache_read"] + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.02805, "input_cost": 0.01785, "output_cost": 0.0102, "prompt_tokens": 150, "completion_tokens": 30}, + "azure/gpt-5.4-mini": {"spend": 0.0264, "input_cost": 0.0168, "output_cost": 0.0096, "prompt_tokens": 150, "completion_tokens": 30}, + "azure/gpt-5.6": {"spend": 0.02475, "input_cost": 0.01575, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-haiku-4-5": {"spend": 0.01155, "input_cost": 0.00735, "output_cost": 0.0042, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-opus-5": {"spend": 0.00825, "input_cost": 0.00525, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-sonnet-5": {"spend": 0.0099, "input_cost": 0.0063, "output_cost": 0.0036, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0231, "input_cost": 0.0147, "output_cost": 0.0084, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/kimi-k3": {"spend": 0.0198, "input_cost": 0.0126, "output_cost": 0.0072, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/qwen3p8-max": {"spend": 0.02145, "input_cost": 0.01365, "output_cost": 0.0078, "prompt_tokens": 150, "completion_tokens": 30}, + "gemini-3.1-pro-preview": {"spend": 0.03465, "input_cost": 0.02205, "output_cost": 0.0126, "prompt_tokens": 150, "completion_tokens": 30}, + "gemini-3.8-flash": {"spend": 0.033, "input_cost": 0.021, "output_cost": 0.012, "prompt_tokens": 150, "completion_tokens": 30}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.01485, "input_cost": 0.00945, "output_cost": 0.0054, "prompt_tokens": 150, "completion_tokens": 30}, + "gemini/gemini-3.8-flash": {"spend": 0.0132, "input_cost": 0.0084, "output_cost": 0.0048, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.3-codex": {"spend": 0.00495, "input_cost": 0.00315, "output_cost": 0.0018, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.4-mini": {"spend": 0.0066, "input_cost": 0.0042, "output_cost": 0.0024, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.5-pro": {"spend": 0.0033, "input_cost": 0.0021, "output_cost": 0.0012, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.6": {"spend": 0.00165, "input_cost": 0.00105, "output_cost": 0.0006, "prompt_tokens": 150, "completion_tokens": 30}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.0165, "input_cost": 0.0105, "output_cost": 0.006, "prompt_tokens": 150, "completion_tokens": 30}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.01815, "input_cost": 0.01155, "output_cost": 0.0066, "prompt_tokens": 150, "completion_tokens": 30}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0297, "input_cost": 0.0189, "output_cost": 0.0108, "prompt_tokens": 150, "completion_tokens": 30} + } }, { "name": "cache_write_5m", "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 60, "output_tokens": 30}, - "requires_rates": ["cache_creation_input_token_cost"], - "requires_caps": ["cache_write_5m"] + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.0561, "input_cost": 0.0459, "output_cost": 0.0102, "prompt_tokens": 150, "completion_tokens": 30}, + "azure/gpt-5.4-mini": {"spend": 0.0528, "input_cost": 0.0432, "output_cost": 0.0096, "prompt_tokens": 150, "completion_tokens": 30}, + "azure/gpt-5.6": {"spend": 0.0495, "input_cost": 0.0405, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-haiku-4-5": {"spend": 0.0231, "input_cost": 0.0189, "output_cost": 0.0042, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-opus-5": {"spend": 0.0165, "input_cost": 0.0135, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-sonnet-5": {"spend": 0.0198, "input_cost": 0.0162, "output_cost": 0.0036, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0408, "input_cost": 0.0324, "output_cost": 0.0084, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/kimi-k3": {"spend": 0.0378, "input_cost": 0.0306, "output_cost": 0.0072, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/qwen3p8-max": {"spend": 0.0393, "input_cost": 0.0315, "output_cost": 0.0078, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.4-mini": {"spend": 0.0132, "input_cost": 0.0108, "output_cost": 0.0024, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.6": {"spend": 0.0033, "input_cost": 0.0027, "output_cost": 0.0006, "prompt_tokens": 150, "completion_tokens": 30}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.033, "input_cost": 0.027, "output_cost": 0.006, "prompt_tokens": 150, "completion_tokens": 30}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0363, "input_cost": 0.0297, "output_cost": 0.0066, "prompt_tokens": 150, "completion_tokens": 30}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0594, "input_cost": 0.0486, "output_cost": 0.0108, "prompt_tokens": 150, "completion_tokens": 30} + } }, { "name": "cache_write_1h", "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 40, "output_tokens": 30}, - "requires_rates": ["cache_creation_input_token_cost_above_1hr", "cache_creation_input_token_cost"], - "requires_caps": ["cache_write_1h"] + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.0629, "input_cost": 0.0527, "output_cost": 0.0102, "prompt_tokens": 150, "completion_tokens": 30}, + "azure/gpt-5.4-mini": {"spend": 0.0592, "input_cost": 0.0496, "output_cost": 0.0096, "prompt_tokens": 150, "completion_tokens": 30}, + "azure/gpt-5.6": {"spend": 0.0555, "input_cost": 0.0465, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-haiku-4-5": {"spend": 0.0259, "input_cost": 0.0217, "output_cost": 0.0042, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-opus-5": {"spend": 0.0185, "input_cost": 0.0155, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 30}, + "claude-sonnet-5": {"spend": 0.0222, "input_cost": 0.0186, "output_cost": 0.0036, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0452, "input_cost": 0.0368, "output_cost": 0.0084, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/kimi-k3": {"spend": 0.0422, "input_cost": 0.035, "output_cost": 0.0072, "prompt_tokens": 150, "completion_tokens": 30}, + "fireworks_ai/qwen3p8-max": {"spend": 0.0437, "input_cost": 0.0359, "output_cost": 0.0078, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.4-mini": {"spend": 0.0148, "input_cost": 0.0124, "output_cost": 0.0024, "prompt_tokens": 150, "completion_tokens": 30}, + "gpt-5.6": {"spend": 0.0037, "input_cost": 0.0031, "output_cost": 0.0006, "prompt_tokens": 150, "completion_tokens": 30}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.037, "input_cost": 0.031, "output_cost": 0.006, "prompt_tokens": 150, "completion_tokens": 30}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0407, "input_cost": 0.0341, "output_cost": 0.0066, "prompt_tokens": 150, "completion_tokens": 30}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0666, "input_cost": 0.0558, "output_cost": 0.0108, "prompt_tokens": 150, "completion_tokens": 30} + } }, { "name": "reasoning", "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "reasoning_tokens": 70}, - "requires_rates": ["output_cost_per_reasoning_token"], - "requires_caps": ["reasoning"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.0816, "input_cost": 0.016, "output_cost": 0.0656, "prompt_tokens": 100, "completion_tokens": 100}, + "azure/gpt-5.6": {"spend": 0.0765, "input_cost": 0.015, "output_cost": 0.0615, "prompt_tokens": 100, "completion_tokens": 100}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0609, "input_cost": 0.014, "output_cost": 0.0469, "prompt_tokens": 100, "completion_tokens": 100}, + "fireworks_ai/kimi-k3": {"spend": 0.0577, "input_cost": 0.012, "output_cost": 0.0457, "prompt_tokens": 100, "completion_tokens": 100}, + "fireworks_ai/qwen3p8-max": {"spend": 0.0593, "input_cost": 0.013, "output_cost": 0.0463, "prompt_tokens": 100, "completion_tokens": 100}, + "gemini-3.1-pro-preview": {"spend": 0.1071, "input_cost": 0.021, "output_cost": 0.0861, "prompt_tokens": 100, "completion_tokens": 100}, + "gemini-3.8-flash": {"spend": 0.102, "input_cost": 0.02, "output_cost": 0.082, "prompt_tokens": 100, "completion_tokens": 100}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.0459, "input_cost": 0.009, "output_cost": 0.0369, "prompt_tokens": 100, "completion_tokens": 100}, + "gemini/gemini-3.8-flash": {"spend": 0.0408, "input_cost": 0.008, "output_cost": 0.0328, "prompt_tokens": 100, "completion_tokens": 100}, + "gpt-5.3-codex": {"spend": 0.0153, "input_cost": 0.003, "output_cost": 0.0123, "prompt_tokens": 100, "completion_tokens": 100}, + "gpt-5.4-mini": {"spend": 0.0204, "input_cost": 0.004, "output_cost": 0.0164, "prompt_tokens": 100, "completion_tokens": 100}, + "gpt-5.5-pro": {"spend": 0.0102, "input_cost": 0.002, "output_cost": 0.0082, "prompt_tokens": 100, "completion_tokens": 100}, + "gpt-5.6": {"spend": 0.0051, "input_cost": 0.001, "output_cost": 0.0041, "prompt_tokens": 100, "completion_tokens": 100}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.051, "input_cost": 0.01, "output_cost": 0.041, "prompt_tokens": 100, "completion_tokens": 100}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0561, "input_cost": 0.011, "output_cost": 0.0451, "prompt_tokens": 100, "completion_tokens": 100} + } }, { "name": "audio", "usage": {"fresh_input_tokens": 100, "audio_input_tokens": 25, "output_tokens": 30, "audio_output_tokens": 15}, - "requires_rates": ["input_cost_per_audio_token", "output_cost_per_audio_token"], - "requires_caps": ["audio"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.0664, "input_cost": 0.04, "output_cost": 0.0264, "prompt_tokens": 125, "completion_tokens": 45}, + "azure/gpt-5.6": {"spend": 0.06225, "input_cost": 0.0375, "output_cost": 0.02475, "prompt_tokens": 125, "completion_tokens": 45}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.05045, "input_cost": 0.0305, "output_cost": 0.01995, "prompt_tokens": 125, "completion_tokens": 45}, + "fireworks_ai/kimi-k3": {"spend": 0.04725, "input_cost": 0.0285, "output_cost": 0.01875, "prompt_tokens": 125, "completion_tokens": 45}, + "fireworks_ai/qwen3p8-max": {"spend": 0.04885, "input_cost": 0.0295, "output_cost": 0.01935, "prompt_tokens": 125, "completion_tokens": 45}, + "gemini-3.1-pro-preview": {"spend": 0.08715, "input_cost": 0.0525, "output_cost": 0.03465, "prompt_tokens": 125, "completion_tokens": 45}, + "gemini-3.8-flash": {"spend": 0.083, "input_cost": 0.05, "output_cost": 0.033, "prompt_tokens": 125, "completion_tokens": 45}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.03735, "input_cost": 0.0225, "output_cost": 0.01485, "prompt_tokens": 125, "completion_tokens": 45}, + "gemini/gemini-3.8-flash": {"spend": 0.0332, "input_cost": 0.02, "output_cost": 0.0132, "prompt_tokens": 125, "completion_tokens": 45}, + "gpt-5.4-mini": {"spend": 0.0166, "input_cost": 0.01, "output_cost": 0.0066, "prompt_tokens": 125, "completion_tokens": 45}, + "gpt-5.6": {"spend": 0.00415, "input_cost": 0.0025, "output_cost": 0.00165, "prompt_tokens": 125, "completion_tokens": 45}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.0415, "input_cost": 0.025, "output_cost": 0.0165, "prompt_tokens": 125, "completion_tokens": 45}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.04565, "input_cost": 0.0275, "output_cost": 0.01815, "prompt_tokens": 125, "completion_tokens": 45} + } }, { "name": "tiered", "usage": {"fresh_input_tokens": 200001, "output_tokens": 30}, - "requires_rates": ["input_cost_per_token_above_200k_tokens", "output_cost_per_token_above_200k_tokens"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 256.04448, "input_cost": 256.00128, "output_cost": 0.0432, "prompt_tokens": 200001, "completion_tokens": 30}, + "azure/gpt-5.6": {"spend": 240.0417, "input_cost": 240.0012, "output_cost": 0.0405, "prompt_tokens": 200001, "completion_tokens": 30}, + "gemini-3.1-pro-preview": {"spend": 336.05838, "input_cost": 336.00168, "output_cost": 0.0567, "prompt_tokens": 200001, "completion_tokens": 30}, + "gemini-3.8-flash": {"spend": 320.0556, "input_cost": 320.0016, "output_cost": 0.054, "prompt_tokens": 200001, "completion_tokens": 30}, + "gemini/gemini-3.1-pro-preview": {"spend": 144.02502, "input_cost": 144.00072, "output_cost": 0.0243, "prompt_tokens": 200001, "completion_tokens": 30}, + "gemini/gemini-3.8-flash": {"spend": 128.02224, "input_cost": 128.00064, "output_cost": 0.0216, "prompt_tokens": 200001, "completion_tokens": 30}, + "gpt-5.3-codex": {"spend": 48.00834, "input_cost": 48.00024, "output_cost": 0.0081, "prompt_tokens": 200001, "completion_tokens": 30}, + "gpt-5.4-mini": {"spend": 64.01112, "input_cost": 64.00032, "output_cost": 0.0108, "prompt_tokens": 200001, "completion_tokens": 30}, + "gpt-5.5-pro": {"spend": 32.00556, "input_cost": 32.00016, "output_cost": 0.0054, "prompt_tokens": 200001, "completion_tokens": 30}, + "gpt-5.6": {"spend": 16.00278, "input_cost": 16.00008, "output_cost": 0.0027, "prompt_tokens": 200001, "completion_tokens": 30}, + "together_ai/moonshotai/Kimi-K3": {"spend": 160.0278, "input_cost": 160.0008, "output_cost": 0.027, "prompt_tokens": 200001, "completion_tokens": 30}, + "together_ai/zai-org/GLM-5.3": {"spend": 176.03058, "input_cost": 176.00088, "output_cost": 0.0297, "prompt_tokens": 200001, "completion_tokens": 30} + } }, { "name": "service_tier_flex", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "service_tier": "flex", - "requires_rates": ["input_cost_per_token_flex", "output_cost_per_token_flex"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.0448, "input_cost": 0.0288, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.042, "input_cost": 0.027, "output_cost": 0.015, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.0588, "input_cost": 0.0378, "output_cost": 0.021, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.056, "input_cost": 0.036, "output_cost": 0.02, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.0252, "input_cost": 0.0162, "output_cost": 0.009, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.0224, "input_cost": 0.0144, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.0084, "input_cost": 0.0054, "output_cost": 0.003, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.0112, "input_cost": 0.0072, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.0056, "input_cost": 0.0036, "output_cost": 0.002, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.0028, "input_cost": 0.0018, "output_cost": 0.001, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.028, "input_cost": 0.018, "output_cost": 0.01, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0308, "input_cost": 0.0198, "output_cost": 0.011, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "service_tier_priority", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "service_tier": "priority", - "requires_rates": ["input_cost_per_token_priority", "output_cost_per_token_priority"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.04992, "input_cost": 0.03264, "output_cost": 0.01728, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.0468, "input_cost": 0.0306, "output_cost": 0.0162, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.06552, "input_cost": 0.04284, "output_cost": 0.02268, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.0624, "input_cost": 0.0408, "output_cost": 0.0216, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.02808, "input_cost": 0.01836, "output_cost": 0.00972, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.02496, "input_cost": 0.01632, "output_cost": 0.00864, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.00936, "input_cost": 0.00612, "output_cost": 0.00324, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.01248, "input_cost": 0.00816, "output_cost": 0.00432, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.00624, "input_cost": 0.00408, "output_cost": 0.00216, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.00312, "input_cost": 0.00204, "output_cost": 0.00108, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.0312, "input_cost": 0.0204, "output_cost": 0.0108, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.03432, "input_cost": 0.02244, "output_cost": 0.01188, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "web_search", "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 3}, - "requires_rates": ["search_context_cost_per_query"], - "requires_caps": ["web_search"], - "wires": ["openai_responses", "anthropic_messages", "gemini_generate", "vertex_generate"] + "expected": { + "claude-haiku-4-5": {"spend": 0.0712, "input_cost": 0.007, "output_cost": 0.0042, "prompt_tokens": 100, "completion_tokens": 30}, + "claude-opus-5": {"spend": 0.068, "input_cost": 0.005, "output_cost": 0.003, "prompt_tokens": 100, "completion_tokens": 30}, + "claude-sonnet-5": {"spend": 0.0696, "input_cost": 0.006, "output_cost": 0.0036, "prompt_tokens": 100, "completion_tokens": 30}, + "gemini-3.1-pro-preview": {"spend": 0.0936, "input_cost": 0.021, "output_cost": 0.0126, "prompt_tokens": 100, "completion_tokens": 30}, + "gemini-3.8-flash": {"spend": 0.092, "input_cost": 0.02, "output_cost": 0.012, "prompt_tokens": 100, "completion_tokens": 30}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.0744, "input_cost": 0.009, "output_cost": 0.0054, "prompt_tokens": 100, "completion_tokens": 30}, + "gemini/gemini-3.8-flash": {"spend": 0.0728, "input_cost": 0.008, "output_cost": 0.0048, "prompt_tokens": 100, "completion_tokens": 30}, + "gpt-5.3-codex": {"spend": 0.0648, "input_cost": 0.003, "output_cost": 0.0018, "prompt_tokens": 100, "completion_tokens": 30}, + "gpt-5.5-pro": {"spend": 0.0632, "input_cost": 0.002, "output_cost": 0.0012, "prompt_tokens": 100, "completion_tokens": 30} + } }, { "name": "web_search_single", "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 1}, - "requires_rates": ["search_context_cost_per_query"], - "requires_caps": ["web_search"], - "wires": ["openai_chat", "together_chat", "fireworks_chat", "azure_chat"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.0456, "input_cost": 0.016, "output_cost": 0.0096, "prompt_tokens": 100, "completion_tokens": 30}, + "azure/gpt-5.6": {"spend": 0.044, "input_cost": 0.015, "output_cost": 0.009, "prompt_tokens": 100, "completion_tokens": 30}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0424, "input_cost": 0.014, "output_cost": 0.0084, "prompt_tokens": 100, "completion_tokens": 30}, + "fireworks_ai/kimi-k3": {"spend": 0.0392, "input_cost": 0.012, "output_cost": 0.0072, "prompt_tokens": 100, "completion_tokens": 30}, + "fireworks_ai/qwen3p8-max": {"spend": 0.0408, "input_cost": 0.013, "output_cost": 0.0078, "prompt_tokens": 100, "completion_tokens": 30}, + "gpt-5.4-mini": {"spend": 0.0264, "input_cost": 0.004, "output_cost": 0.0024, "prompt_tokens": 100, "completion_tokens": 30}, + "gpt-5.6": {"spend": 0.0216, "input_cost": 0.001, "output_cost": 0.0006, "prompt_tokens": 100, "completion_tokens": 30}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.036, "input_cost": 0.01, "output_cost": 0.006, "prompt_tokens": 100, "completion_tokens": 30}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0376, "input_cost": 0.011, "output_cost": 0.0066, "prompt_tokens": 100, "completion_tokens": 30} + } }, { "name": "stream", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, - "stream": true + "stream": true, + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.034, "input_cost": 0.0204, "output_cost": 0.0136, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.03, "input_cost": 0.018, "output_cost": 0.012, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-haiku-4-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-opus-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-sonnet-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/kimi-k3": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/qwen3p8-max": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, + "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0228, "output_cost": 0.0152, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.036, "input_cost": 0.0216, "output_cost": 0.0144, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "stream_no_usage", @@ -83,33 +256,137 @@ "stream": true, "stream_usage": "absent", "exact_spend": false, - "requires_caps": ["absent_usage"] + "models": [ + "anthropic.claude-sonnet-5-v1:0", + "azure/gpt-5.4-mini", + "azure/gpt-5.6", + "claude-haiku-4-5", + "claude-opus-5", + "claude-sonnet-5", + "fireworks_ai/deepseek-v4p1-flash", + "fireworks_ai/kimi-k3", + "fireworks_ai/qwen3p8-max", + "gemini-3.1-pro-preview", + "gemini-3.8-flash", + "gemini/gemini-3.1-pro-preview", + "gemini/gemini-3.8-flash", + "gpt-5.3-codex", + "gpt-5.4-mini", + "gpt-5.5-pro", + "gpt-5.6", + "meta.llama4-maverick-17b-instruct-v1:0", + "together_ai/moonshotai/Kimi-K3", + "together_ai/zai-org/GLM-5.3", + "us.anthropic.claude-opus-5-v1:0" + ] }, { "name": "response_model_override", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "response_model_override": true, - "requires_caps": ["response_model"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-haiku-4-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-opus-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-sonnet-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/kimi-k3": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/qwen3p8-max": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "stream_response_model_override", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "stream": true, "response_model_override": true, - "requires_caps": ["response_model"] + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-haiku-4-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-opus-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-sonnet-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/kimi-k3": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/qwen3p8-max": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "tool_call", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "tool_call": true, - "requires_caps": ["tool_call"] + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.034, "input_cost": 0.0204, "output_cost": 0.0136, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, + "azure/gpt-5.6": {"spend": 0.03, "input_cost": 0.018, "output_cost": 0.012, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-haiku-4-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-opus-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, + "claude-sonnet-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/kimi-k3": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, + "fireworks_ai/qwen3p8-max": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.1-pro-preview": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini-3.8-flash": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, + "gemini/gemini-3.8-flash": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.4-mini": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.6": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, + "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0228, "output_cost": 0.0152, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.036, "input_cost": 0.0216, "output_cost": 0.0144, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "stream_tool_call", "usage": {"fresh_input_tokens": 80, "output_tokens": 25}, "stream": true, "tool_call": true, - "requires_caps": ["tool_call"] + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.0221, "input_cost": 0.0136, "output_cost": 0.0085, "prompt_tokens": 80, "completion_tokens": 25}, + "azure/gpt-5.4-mini": {"spend": 0.0208, "input_cost": 0.0128, "output_cost": 0.008, "prompt_tokens": 80, "completion_tokens": 25}, + "azure/gpt-5.6": {"spend": 0.0195, "input_cost": 0.012, "output_cost": 0.0075, "prompt_tokens": 80, "completion_tokens": 25}, + "claude-haiku-4-5": {"spend": 0.0091, "input_cost": 0.0056, "output_cost": 0.0035, "prompt_tokens": 80, "completion_tokens": 25}, + "claude-opus-5": {"spend": 0.0065, "input_cost": 0.004, "output_cost": 0.0025, "prompt_tokens": 80, "completion_tokens": 25}, + "claude-sonnet-5": {"spend": 0.0078, "input_cost": 0.0048, "output_cost": 0.003, "prompt_tokens": 80, "completion_tokens": 25}, + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0182, "input_cost": 0.0112, "output_cost": 0.007, "prompt_tokens": 80, "completion_tokens": 25}, + "fireworks_ai/kimi-k3": {"spend": 0.0156, "input_cost": 0.0096, "output_cost": 0.006, "prompt_tokens": 80, "completion_tokens": 25}, + "fireworks_ai/qwen3p8-max": {"spend": 0.0169, "input_cost": 0.0104, "output_cost": 0.0065, "prompt_tokens": 80, "completion_tokens": 25}, + "gemini-3.1-pro-preview": {"spend": 0.0273, "input_cost": 0.0168, "output_cost": 0.0105, "prompt_tokens": 80, "completion_tokens": 25}, + "gemini-3.8-flash": {"spend": 0.026, "input_cost": 0.016, "output_cost": 0.01, "prompt_tokens": 80, "completion_tokens": 25}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.0117, "input_cost": 0.0072, "output_cost": 0.0045, "prompt_tokens": 80, "completion_tokens": 25}, + "gemini/gemini-3.8-flash": {"spend": 0.0104, "input_cost": 0.0064, "output_cost": 0.004, "prompt_tokens": 80, "completion_tokens": 25}, + "gpt-5.3-codex": {"spend": 0.0039, "input_cost": 0.0024, "output_cost": 0.0015, "prompt_tokens": 80, "completion_tokens": 25}, + "gpt-5.4-mini": {"spend": 0.0052, "input_cost": 0.0032, "output_cost": 0.002, "prompt_tokens": 80, "completion_tokens": 25}, + "gpt-5.5-pro": {"spend": 0.0026, "input_cost": 0.0016, "output_cost": 0.001, "prompt_tokens": 80, "completion_tokens": 25}, + "gpt-5.6": {"spend": 0.0013, "input_cost": 0.0008, "output_cost": 0.0005, "prompt_tokens": 80, "completion_tokens": 25}, + "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.0247, "input_cost": 0.0152, "output_cost": 0.0095, "prompt_tokens": 80, "completion_tokens": 25}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.013, "input_cost": 0.008, "output_cost": 0.005, "prompt_tokens": 80, "completion_tokens": 25}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0143, "input_cost": 0.0088, "output_cost": 0.0055, "prompt_tokens": 80, "completion_tokens": 25}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0234, "input_cost": 0.0144, "output_cost": 0.009, "prompt_tokens": 80, "completion_tokens": 25} + } }, { "name": "stream_no_usage_tool_call", @@ -118,7 +395,29 @@ "stream_usage": "absent", "tool_call": true, "exact_spend": false, - "requires_caps": ["absent_usage", "tool_call"] + "models": [ + "anthropic.claude-sonnet-5-v1:0", + "azure/gpt-5.4-mini", + "azure/gpt-5.6", + "claude-haiku-4-5", + "claude-opus-5", + "claude-sonnet-5", + "fireworks_ai/deepseek-v4p1-flash", + "fireworks_ai/kimi-k3", + "fireworks_ai/qwen3p8-max", + "gemini-3.1-pro-preview", + "gemini-3.8-flash", + "gemini/gemini-3.1-pro-preview", + "gemini/gemini-3.8-flash", + "gpt-5.3-codex", + "gpt-5.4-mini", + "gpt-5.5-pro", + "gpt-5.6", + "meta.llama4-maverick-17b-instruct-v1:0", + "together_ai/moonshotai/Kimi-K3", + "together_ai/zai-org/GLM-5.3", + "us.anthropic.claude-opus-5-v1:0" + ] }, { "name": "stream_no_usage_image_input", @@ -127,14 +426,39 @@ "stream_usage": "absent", "image_input": true, "exact_spend": false, - "requires_caps": ["absent_usage", "image_input"] + "models": [ + "anthropic.claude-sonnet-5-v1:0", + "azure/gpt-5.4-mini", + "azure/gpt-5.6", + "claude-haiku-4-5", + "claude-opus-5", + "claude-sonnet-5", + "fireworks_ai/deepseek-v4p1-flash", + "fireworks_ai/kimi-k3", + "fireworks_ai/qwen3p8-max", + "gemini-3.1-pro-preview", + "gemini-3.8-flash", + "gemini/gemini-3.1-pro-preview", + "gemini/gemini-3.8-flash", + "gpt-5.3-codex", + "gpt-5.4-mini", + "gpt-5.5-pro", + "gpt-5.6", + "meta.llama4-maverick-17b-instruct-v1:0", + "together_ai/moonshotai/Kimi-K3", + "together_ai/zai-org/GLM-5.3", + "us.anthropic.claude-opus-5-v1:0" + ] }, { "name": "stream_incomplete", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "stream": true, "terminal": "incomplete", - "requires_caps": ["responses_terminal"] + "expected": { + "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "stream_no_usage_incomplete", @@ -143,14 +467,20 @@ "stream_usage": "absent", "terminal": "incomplete", "exact_spend": false, - "requires_caps": ["responses_terminal"] + "models": [ + "gpt-5.3-codex", + "gpt-5.5-pro" + ] }, { "name": "stream_unvalidated", "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, "stream": true, "terminal": "unvalidated", - "requires_caps": ["responses_terminal"] + "expected": { + "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40} + } }, { "name": "stream_no_usage_unvalidated", @@ -159,14 +489,22 @@ "stream_usage": "absent", "terminal": "unvalidated", "exact_spend": false, - "requires_caps": ["responses_terminal"] + "models": [ + "gpt-5.3-codex", + "gpt-5.5-pro" + ] }, { "name": "prompt_blocked", "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, "terminal": "prompt_blocked", "response_model_override": true, - "requires_caps": ["prompt_blocked"] + "expected": { + "gemini-3.1-pro-preview": {"spend": 0.2, "input_cost": 0.2, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, + "gemini-3.8-flash": {"spend": 0.21, "input_cost": 0.21, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.08, "input_cost": 0.08, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, + "gemini/gemini-3.8-flash": {"spend": 0.09, "input_cost": 0.09, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0} + } }, { "name": "stream_prompt_blocked", @@ -174,77 +512,73 @@ "stream": true, "terminal": "prompt_blocked", "response_model_override": true, - "requires_caps": ["prompt_blocked"] + "expected": { + "gemini-3.1-pro-preview": {"spend": 0.2, "input_cost": 0.2, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, + "gemini-3.8-flash": {"spend": 0.21, "input_cost": 0.21, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.08, "input_cost": 0.08, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, + "gemini/gemini-3.8-flash": {"spend": 0.09, "input_cost": 0.09, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0} + } }, { "name": "all_components_chat", - "usage": { - "fresh_input_tokens": 80, - "cache_read_tokens": 40, - "cache_write_5m_tokens": 20, - "cache_write_1h_tokens": 10, - "output_tokens": 25, - "reasoning_tokens": 15, - "audio_input_tokens": 5, - "audio_output_tokens": 3 - }, - "requires_rates": [ - "output_cost_per_reasoning_token", - "input_cost_per_audio_token", - "output_cost_per_audio_token" - ], - "wires": ["openai_chat", "azure_chat", "together_chat"] + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 10, "output_tokens": 25, "reasoning_tokens": 15, "audio_input_tokens": 5, "audio_output_tokens": 3}, + "expected": { + "azure/gpt-5.4-mini": {"spend": 0.0576, "input_cost": 0.03424, "output_cost": 0.02336, "prompt_tokens": 155, "completion_tokens": 43}, + "azure/gpt-5.6": {"spend": 0.054, "input_cost": 0.0321, "output_cost": 0.0219, "prompt_tokens": 155, "completion_tokens": 43}, + "gpt-5.4-mini": {"spend": 0.0144, "input_cost": 0.00856, "output_cost": 0.00584, "prompt_tokens": 155, "completion_tokens": 43}, + "gpt-5.6": {"spend": 0.0036, "input_cost": 0.00214, "output_cost": 0.00146, "prompt_tokens": 155, "completion_tokens": 43}, + "together_ai/moonshotai/Kimi-K3": {"spend": 0.036, "input_cost": 0.0214, "output_cost": 0.0146, "prompt_tokens": 155, "completion_tokens": 43}, + "together_ai/zai-org/GLM-5.3": {"spend": 0.0396, "input_cost": 0.02354, "output_cost": 0.01606, "prompt_tokens": 155, "completion_tokens": 43} + } }, { "name": "all_components_fireworks", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, - "wires": ["fireworks_chat"] + "expected": { + "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.01876, "input_cost": 0.01176, "output_cost": 0.007, "prompt_tokens": 120, "completion_tokens": 25}, + "fireworks_ai/kimi-k3": {"spend": 0.01608, "input_cost": 0.01008, "output_cost": 0.006, "prompt_tokens": 120, "completion_tokens": 25}, + "fireworks_ai/qwen3p8-max": {"spend": 0.01742, "input_cost": 0.01092, "output_cost": 0.0065, "prompt_tokens": 120, "completion_tokens": 25} + } }, { "name": "all_components_anthropic", - "usage": { - "fresh_input_tokens": 80, - "cache_read_tokens": 40, - "cache_write_5m_tokens": 20, - "cache_write_1h_tokens": 10, - "output_tokens": 25 - }, - "wires": ["anthropic_messages", "bedrock_converse"] + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 10, "output_tokens": 25}, + "expected": { + "anthropic.claude-sonnet-5-v1:0": {"spend": 0.03978, "input_cost": 0.03128, "output_cost": 0.0085, "prompt_tokens": 150, "completion_tokens": 25}, + "claude-haiku-4-5": {"spend": 0.01638, "input_cost": 0.01288, "output_cost": 0.0035, "prompt_tokens": 150, "completion_tokens": 25}, + "claude-opus-5": {"spend": 0.0117, "input_cost": 0.0092, "output_cost": 0.0025, "prompt_tokens": 150, "completion_tokens": 25}, + "claude-sonnet-5": {"spend": 0.01404, "input_cost": 0.01104, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 25}, + "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0285, "output_cost": 0.0095, "prompt_tokens": 150, "completion_tokens": 25}, + "us.anthropic.claude-opus-5-v1:0": {"spend": 0.04212, "input_cost": 0.03312, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 25} + } }, { "name": "all_components_anthropic_stream", - "usage": { - "fresh_input_tokens": 80, - "cache_read_tokens": 40, - "cache_write_5m_tokens": 20, - "cache_write_1h_tokens": 10, - "output_tokens": 25 - }, + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 10, "output_tokens": 25}, "stream": true, - "wires": ["anthropic_messages"] + "expected": { + "claude-haiku-4-5": {"spend": 0.01638, "input_cost": 0.01288, "output_cost": 0.0035, "prompt_tokens": 150, "completion_tokens": 25}, + "claude-opus-5": {"spend": 0.0117, "input_cost": 0.0092, "output_cost": 0.0025, "prompt_tokens": 150, "completion_tokens": 25}, + "claude-sonnet-5": {"spend": 0.01404, "input_cost": 0.01104, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 25} + } }, { "name": "all_components_gemini", - "usage": { - "fresh_input_tokens": 80, - "cache_read_tokens": 40, - "output_tokens": 25, - "reasoning_tokens": 15, - "audio_input_tokens": 5, - "audio_output_tokens": 3 - }, - "requires_rates": [ - "output_cost_per_reasoning_token", - "input_cost_per_audio_token", - "output_cost_per_audio_token" - ], - "wires": ["gemini_generate", "vertex_generate"] + "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15, "audio_input_tokens": 5, "audio_output_tokens": 3}, + "expected": { + "gemini-3.1-pro-preview": {"spend": 0.0546, "input_cost": 0.02394, "output_cost": 0.03066, "prompt_tokens": 125, "completion_tokens": 43}, + "gemini-3.8-flash": {"spend": 0.052, "input_cost": 0.0228, "output_cost": 0.0292, "prompt_tokens": 125, "completion_tokens": 43}, + "gemini/gemini-3.1-pro-preview": {"spend": 0.0234, "input_cost": 0.01026, "output_cost": 0.01314, "prompt_tokens": 125, "completion_tokens": 43}, + "gemini/gemini-3.8-flash": {"spend": 0.0208, "input_cost": 0.00912, "output_cost": 0.01168, "prompt_tokens": 125, "completion_tokens": 43} + } }, { "name": "all_components_responses", "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, - "requires_rates": ["output_cost_per_reasoning_token"], - "wires": ["openai_responses"] + "expected": { + "gpt-5.3-codex": {"spend": 0.00627, "input_cost": 0.00252, "output_cost": 0.00375, "prompt_tokens": 120, "completion_tokens": 40}, + "gpt-5.5-pro": {"spend": 0.00418, "input_cost": 0.00168, "output_cost": 0.0025, "prompt_tokens": 120, "completion_tokens": 40} + } } ] } diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 3de9786854e..1473edb119b 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -2,9 +2,8 @@ Runs against a dedicated proxy whose whole model cost map is the test-owned ``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL); every map entry is a -deployment under test, the request shapes live in ``cases.json``, and the -asserted goldens live in ``expected.json`` (regenerate proposals with -``generate_expected.py``). Provider calls are answered by the +deployment under test, and the request shapes plus asserted goldens live in +``cases.json``. Provider calls are answered by the scripted-provider sidecar (``scripted_provider.py``), registered per scenario over its control API. diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 68f3186809d..7999d827060 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -1,16 +1,13 @@ """The cost-calculation matrix: the model set derived from the test cost map, the request/response cases from ``cases.json``, and the loaders both use. -Three data files drive the suite; nothing in Python lists models or cases: +Two data files drive the suite; nothing in Python lists models or cases: - ``tests/e2e/cost_map.json`` is the proxy's ENTIRE model cost map (LITELLM_MODEL_COST_MAP_URL); every entry becomes a deployment under test. -- ``tests/e2e/cost_calculation/cases.json`` is the case list; each case runs - for a model when the entry carries the rates it exercises (``requires_rates``) - and the wire can report the token kinds involved (``requires_caps`` / - ``wires``). -- ``tests/e2e/cost_calculation/expected.json`` holds the reviewed goldens; the - tests assert them verbatim and never compute a price themselves. The rate - arithmetic that proposes goldens lives in ``generate_expected.py``, not here. +- ``tests/e2e/cost_calculation/cases.json`` is the case list plus the reviewed + goldens: each exact-spend case carries an ``expected`` cell per map key it + runs against, each recount case carries its ``models`` list, so matrix + membership and expected values are literal data read side by side. """ from __future__ import annotations @@ -26,13 +23,11 @@ from pathlib import Path from types import MappingProxyType from typing import Final, Literal -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" -EXPECTED_PATH: Final = Path(__file__).resolve().parent / "expected.json" - class SearchContextCostPerQuery(BaseModel): model_config = ConfigDict(frozen=True) @@ -88,10 +83,20 @@ class DeploymentSpec(BaseModel): base_model: str | None = None +class ExpectedCell(BaseModel): + model_config = ConfigDict(frozen=True) + + spend: float + input_cost: float + output_cost: float + prompt_tokens: int + completion_tokens: int + + class Case(BaseModel): - """One request/response shape from cases.json; gated onto a model by - ``requires_rates`` (entry must carry each rate field), ``requires_caps`` - (the wire must report the token kind) and ``wires`` (shape is wire-specific).""" + """One request/response shape from cases.json. An exact-spend case names + its models implicitly by carrying one ``expected`` golden per map key; a + recount case (``exact_spend=False``) names them in ``models`` instead.""" model_config = ConfigDict(frozen=True) @@ -105,19 +110,16 @@ class Case(BaseModel): tool_call: bool = False image_input: bool = False terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" - requires_rates: tuple[str, ...] = () - requires_caps: tuple[str, ...] = () - wires: tuple[Wire, ...] | None = None + expected: Mapping[str, ExpectedCell] = Field(default_factory=lambda: MappingProxyType({})) + models: tuple[str, ...] = () def applies_to(self, model: FrontierModel) -> bool: - if self.wires is not None and model.wire not in self.wires: - return False - caps: Final = _WIRE_CAPS[model.wire] - if not frozenset(self.requires_caps) <= caps: - return False - return all( - getattr(model.rates, field, None) is not None for field in self.requires_rates - ) + if self.exact_spend: + return model.map_key in self.expected + return model.map_key in self.models + + def expected_for(self, model: FrontierModel) -> ExpectedCell: + return self.expected[model.map_key] def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( @@ -309,64 +311,6 @@ def _frontier() -> tuple[FrontierModel, ...]: FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() -# Token kinds each wire can report, gating which pricing cases apply. -_WIRE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType({ - "openai_chat": frozenset( - { - "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", "tool_call", "image_input", - } - ), - "openai_responses": frozenset( - { - "cache_read", "reasoning", "web_search", "response_model", "absent_usage", - "tool_call", "image_input", "responses_terminal", - } - ), - "anthropic_messages": frozenset( - { - "cache_read", "cache_write_5m", "cache_write_1h", "web_search", - "response_model", "absent_usage", "tool_call", "image_input", - } - ), - "gemini_generate": frozenset( - { - "cache_read", "reasoning", "audio", "web_search", "response_model", - "absent_usage", "tool_call", "image_input", "prompt_blocked", - } - ), - "together_chat": frozenset( - { - "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", "tool_call", "image_input", - } - ), - "fireworks_chat": frozenset( - { - "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", "tool_call", "image_input", - } - ), - "azure_chat": frozenset( - { - "cache_read", "cache_write_5m", "cache_write_1h", "reasoning", "audio", - "web_search", "response_model", "absent_usage", "tool_call", "image_input", - } - ), - "bedrock_converse": frozenset( - { - "cache_read", "cache_write_5m", "cache_write_1h", "absent_usage", - "tool_call", "image_input", - } - ), - "vertex_generate": frozenset( - { - "cache_read", "reasoning", "audio", "web_search", "response_model", - "absent_usage", "tool_call", "image_input", "prompt_blocked", - } - ), -}) - TOOL_CALL_ARGUMENTS: Final = json.dumps({ "city": "Berlin", "days": 7, @@ -415,64 +359,43 @@ def image_input_data_url() -> str: IMAGE_INPUT_DATA_URL: Final = image_input_data_url() -class ExpectedCell(BaseModel): - model_config = ConfigDict(frozen=True) - - spend: float - input_cost: float - output_cost: float - prompt_tokens: int - completion_tokens: int - - -_EXPECTED_ADAPTER: Final = TypeAdapter(dict[str, ExpectedCell]) -EXPECTED: Final[Mapping[str, ExpectedCell]] = MappingProxyType( - _EXPECTED_ADAPTER.validate_python(json.loads(EXPECTED_PATH.read_text())) - if EXPECTED_PATH.exists() - else {} -) - - -def expected_key(model: FrontierModel, case: Case) -> str: - return f"{model.map_key}|{case.name}" - - def matrix_data_errors() -> tuple[str, ...]: - """Freshness findings for the data files, as human-readable strings. + """Consistency findings for the data files, as human-readable strings. - Called at collection time by the e2e suite; also usable from - generate_expected.py's context without importing pytest. + Called at collection time by the e2e suite, so a map key named by a case + but absent from cost_map.json fails the suite's collection loudly. """ - derived: Final = { - expected_key(model, case) - for model in FRONTIER_MODELS - for case in cases_for(model) - if case.exact_spend - } - golden: Final = set(EXPECTED) unknown_deployments: Final = sorted( spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP ) - unknown_rates: Final = sorted( - {field for case in CASES for field in case.requires_rates} - set(CostMapEntry.model_fields) + unknown_case_models: Final = sorted( + { + map_key + for case in CASES + for map_key in (*case.expected, *case.models) + if map_key not in COST_MAP + } + ) + misshapen_cases: Final = sorted( + case.name + for case in CASES + if case.exact_spend == bool(case.models) or case.exact_spend != bool(case.expected) ) input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) findings: Final = ( - ( - "expected.json is out of sync with the derived matrix; run " - "uv run python tests/e2e/cost_calculation/generate_expected.py " - f"(missing: {sorted(derived - golden)}; stale: {sorted(golden - derived)})" - ) - if derived != golden - else None, ( f"deployments entries name map keys absent from cost_map.json: {unknown_deployments}" if unknown_deployments else None ), ( - f"requires_rates names that are not CostMapEntry fields: {unknown_rates}" - if unknown_rates + f"case expected/models name map keys absent from cost_map.json: {unknown_case_models}" + if unknown_case_models + else None + ), + ( + f"cases must carry expected xor models (exact_spend matches the field): {misshapen_cases}" + if misshapen_cases else None ), ( diff --git a/tests/e2e/cost_calculation/expected.json b/tests/e2e/cost_calculation/expected.json deleted file mode 100644 index 984b670a82c..00000000000 --- a/tests/e2e/cost_calculation/expected.json +++ /dev/null @@ -1,2004 +0,0 @@ -{ - "anthropic.claude-sonnet-5-v1:0|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.03128, - "output_cost": 0.0085, - "prompt_tokens": 150, - "spend": 0.03978 - }, - "anthropic.claude-sonnet-5-v1:0|basic": { - "completion_tokens": 40, - "input_cost": 0.0204, - "output_cost": 0.013600000000000001, - "prompt_tokens": 120, - "spend": 0.034 - }, - "anthropic.claude-sonnet-5-v1:0|cache_read": { - "completion_tokens": 30, - "input_cost": 0.01785, - "output_cost": 0.0102, - "prompt_tokens": 150, - "spend": 0.028050000000000002 - }, - "anthropic.claude-sonnet-5-v1:0|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.052700000000000004, - "output_cost": 0.0102, - "prompt_tokens": 150, - "spend": 0.06290000000000001 - }, - "anthropic.claude-sonnet-5-v1:0|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0459, - "output_cost": 0.0102, - "prompt_tokens": 150, - "spend": 0.056100000000000004 - }, - "anthropic.claude-sonnet-5-v1:0|stream": { - "completion_tokens": 40, - "input_cost": 0.0204, - "output_cost": 0.013600000000000001, - "prompt_tokens": 120, - "spend": 0.034 - }, - "anthropic.claude-sonnet-5-v1:0|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.013600000000000001, - "output_cost": 0.0085, - "prompt_tokens": 80, - "spend": 0.0221 - }, - "anthropic.claude-sonnet-5-v1:0|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0204, - "output_cost": 0.013600000000000001, - "prompt_tokens": 120, - "spend": 0.034 - }, - "azure/gpt-5.4-mini|all_components_chat": { - "completion_tokens": 43, - "input_cost": 0.03424, - "output_cost": 0.02336, - "prompt_tokens": 155, - "spend": 0.0576 - }, - "azure/gpt-5.4-mini|audio": { - "completion_tokens": 45, - "input_cost": 0.04, - "output_cost": 0.0264, - "prompt_tokens": 125, - "spend": 0.0664 - }, - "azure/gpt-5.4-mini|basic": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.4-mini|cache_read": { - "completion_tokens": 30, - "input_cost": 0.0168, - "output_cost": 0.009600000000000001, - "prompt_tokens": 150, - "spend": 0.0264 - }, - "azure/gpt-5.4-mini|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.049600000000000005, - "output_cost": 0.009600000000000001, - "prompt_tokens": 150, - "spend": 0.0592 - }, - "azure/gpt-5.4-mini|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0432, - "output_cost": 0.009600000000000001, - "prompt_tokens": 150, - "spend": 0.0528 - }, - "azure/gpt-5.4-mini|reasoning": { - "completion_tokens": 100, - "input_cost": 0.016, - "output_cost": 0.0656, - "prompt_tokens": 100, - "spend": 0.0816 - }, - "azure/gpt-5.4-mini|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.4-mini|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0288, - "output_cost": 0.016, - "prompt_tokens": 120, - "spend": 0.0448 - }, - "azure/gpt-5.4-mini|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.03264, - "output_cost": 0.01728, - "prompt_tokens": 120, - "spend": 0.049920000000000006 - }, - "azure/gpt-5.4-mini|stream": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.4-mini|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.4-mini|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0128, - "output_cost": 0.008, - "prompt_tokens": 80, - "spend": 0.0208 - }, - "azure/gpt-5.4-mini|tiered": { - "completion_tokens": 30, - "input_cost": 256.00128, - "output_cost": 0.0432, - "prompt_tokens": 200001, - "spend": 256.04448 - }, - "azure/gpt-5.4-mini|tool_call": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.4-mini|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.016, - "output_cost": 0.009600000000000001, - "prompt_tokens": 100, - "spend": 0.0456 - }, - "azure/gpt-5.6|all_components_chat": { - "completion_tokens": 43, - "input_cost": 0.0321, - "output_cost": 0.0219, - "prompt_tokens": 155, - "spend": 0.05399999999999999 - }, - "azure/gpt-5.6|audio": { - "completion_tokens": 45, - "input_cost": 0.0375, - "output_cost": 0.02475, - "prompt_tokens": 125, - "spend": 0.06225 - }, - "azure/gpt-5.6|basic": { - "completion_tokens": 40, - "input_cost": 0.018, - "output_cost": 0.011999999999999999, - "prompt_tokens": 120, - "spend": 0.03 - }, - "azure/gpt-5.6|cache_read": { - "completion_tokens": 30, - "input_cost": 0.01575, - "output_cost": 0.009, - "prompt_tokens": 150, - "spend": 0.02475 - }, - "azure/gpt-5.6|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.0465, - "output_cost": 0.009, - "prompt_tokens": 150, - "spend": 0.0555 - }, - "azure/gpt-5.6|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.040499999999999994, - "output_cost": 0.009, - "prompt_tokens": 150, - "spend": 0.049499999999999995 - }, - "azure/gpt-5.6|reasoning": { - "completion_tokens": 100, - "input_cost": 0.015, - "output_cost": 0.0615, - "prompt_tokens": 100, - "spend": 0.0765 - }, - "azure/gpt-5.6|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.6|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.027, - "output_cost": 0.015, - "prompt_tokens": 120, - "spend": 0.041999999999999996 - }, - "azure/gpt-5.6|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.030600000000000002, - "output_cost": 0.0162, - "prompt_tokens": 120, - "spend": 0.0468 - }, - "azure/gpt-5.6|stream": { - "completion_tokens": 40, - "input_cost": 0.018, - "output_cost": 0.011999999999999999, - "prompt_tokens": 120, - "spend": 0.03 - }, - "azure/gpt-5.6|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.019200000000000002, - "output_cost": 0.0128, - "prompt_tokens": 120, - "spend": 0.032 - }, - "azure/gpt-5.6|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.011999999999999999, - "output_cost": 0.0075, - "prompt_tokens": 80, - "spend": 0.019499999999999997 - }, - "azure/gpt-5.6|tiered": { - "completion_tokens": 30, - "input_cost": 240.00119999999998, - "output_cost": 0.0405, - "prompt_tokens": 200001, - "spend": 240.0417 - }, - "azure/gpt-5.6|tool_call": { - "completion_tokens": 40, - "input_cost": 0.018, - "output_cost": 0.011999999999999999, - "prompt_tokens": 120, - "spend": 0.03 - }, - "azure/gpt-5.6|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.015, - "output_cost": 0.009, - "prompt_tokens": 100, - "spend": 0.044 - }, - "claude-haiku-4-5|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.012880000000000003, - "output_cost": 0.0035000000000000005, - "prompt_tokens": 150, - "spend": 0.016380000000000002 - }, - "claude-haiku-4-5|all_components_anthropic_stream": { - "completion_tokens": 25, - "input_cost": 0.012880000000000003, - "output_cost": 0.0035000000000000005, - "prompt_tokens": 150, - "spend": 0.016380000000000002 - }, - "claude-haiku-4-5|basic": { - "completion_tokens": 40, - "input_cost": 0.008400000000000001, - "output_cost": 0.005600000000000001, - "prompt_tokens": 120, - "spend": 0.014000000000000002 - }, - "claude-haiku-4-5|cache_read": { - "completion_tokens": 30, - "input_cost": 0.007350000000000001, - "output_cost": 0.004200000000000001, - "prompt_tokens": 150, - "spend": 0.011550000000000001 - }, - "claude-haiku-4-5|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.021700000000000004, - "output_cost": 0.004200000000000001, - "prompt_tokens": 150, - "spend": 0.025900000000000006 - }, - "claude-haiku-4-5|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0189, - "output_cost": 0.004200000000000001, - "prompt_tokens": 150, - "spend": 0.023100000000000002 - }, - "claude-haiku-4-5|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.006, - "output_cost": 0.004, - "prompt_tokens": 120, - "spend": 0.01 - }, - "claude-haiku-4-5|stream": { - "completion_tokens": 40, - "input_cost": 0.008400000000000001, - "output_cost": 0.005600000000000001, - "prompt_tokens": 120, - "spend": 0.014000000000000002 - }, - "claude-haiku-4-5|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.006, - "output_cost": 0.004, - "prompt_tokens": 120, - "spend": 0.01 - }, - "claude-haiku-4-5|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.005600000000000001, - "output_cost": 0.0035000000000000005, - "prompt_tokens": 80, - "spend": 0.0091 - }, - "claude-haiku-4-5|tool_call": { - "completion_tokens": 40, - "input_cost": 0.008400000000000001, - "output_cost": 0.005600000000000001, - "prompt_tokens": 120, - "spend": 0.014000000000000002 - }, - "claude-haiku-4-5|web_search": { - "completion_tokens": 30, - "input_cost": 0.007000000000000001, - "output_cost": 0.004200000000000001, - "prompt_tokens": 100, - "spend": 0.0712 - }, - "claude-opus-5|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.0092, - "output_cost": 0.0025, - "prompt_tokens": 150, - "spend": 0.0117 - }, - "claude-opus-5|all_components_anthropic_stream": { - "completion_tokens": 25, - "input_cost": 0.0092, - "output_cost": 0.0025, - "prompt_tokens": 150, - "spend": 0.0117 - }, - "claude-opus-5|basic": { - "completion_tokens": 40, - "input_cost": 0.006, - "output_cost": 0.004, - "prompt_tokens": 120, - "spend": 0.01 - }, - "claude-opus-5|cache_read": { - "completion_tokens": 30, - "input_cost": 0.00525, - "output_cost": 0.003, - "prompt_tokens": 150, - "spend": 0.00825 - }, - "claude-opus-5|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.0155, - "output_cost": 0.003, - "prompt_tokens": 150, - "spend": 0.0185 - }, - "claude-opus-5|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.013500000000000002, - "output_cost": 0.003, - "prompt_tokens": 150, - "spend": 0.0165 - }, - "claude-opus-5|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.007200000000000001, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 120, - "spend": 0.012 - }, - "claude-opus-5|stream": { - "completion_tokens": 40, - "input_cost": 0.006, - "output_cost": 0.004, - "prompt_tokens": 120, - "spend": 0.01 - }, - "claude-opus-5|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.007200000000000001, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 120, - "spend": 0.012 - }, - "claude-opus-5|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.004, - "output_cost": 0.0025, - "prompt_tokens": 80, - "spend": 0.006500000000000001 - }, - "claude-opus-5|tool_call": { - "completion_tokens": 40, - "input_cost": 0.006, - "output_cost": 0.004, - "prompt_tokens": 120, - "spend": 0.01 - }, - "claude-opus-5|web_search": { - "completion_tokens": 30, - "input_cost": 0.005, - "output_cost": 0.003, - "prompt_tokens": 100, - "spend": 0.068 - }, - "claude-sonnet-5|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.011040000000000001, - "output_cost": 0.0030000000000000005, - "prompt_tokens": 150, - "spend": 0.014040000000000002 - }, - "claude-sonnet-5|all_components_anthropic_stream": { - "completion_tokens": 25, - "input_cost": 0.011040000000000001, - "output_cost": 0.0030000000000000005, - "prompt_tokens": 150, - "spend": 0.014040000000000002 - }, - "claude-sonnet-5|basic": { - "completion_tokens": 40, - "input_cost": 0.007200000000000001, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 120, - "spend": 0.012 - }, - "claude-sonnet-5|cache_read": { - "completion_tokens": 30, - "input_cost": 0.006300000000000001, - "output_cost": 0.0036000000000000003, - "prompt_tokens": 150, - "spend": 0.0099 - }, - "claude-sonnet-5|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.018600000000000002, - "output_cost": 0.0036000000000000003, - "prompt_tokens": 150, - "spend": 0.0222 - }, - "claude-sonnet-5|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.016200000000000003, - "output_cost": 0.0036000000000000003, - "prompt_tokens": 150, - "spend": 0.0198 - }, - "claude-sonnet-5|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.008400000000000001, - "output_cost": 0.005600000000000001, - "prompt_tokens": 120, - "spend": 0.014000000000000002 - }, - "claude-sonnet-5|stream": { - "completion_tokens": 40, - "input_cost": 0.007200000000000001, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 120, - "spend": 0.012 - }, - "claude-sonnet-5|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.008400000000000001, - "output_cost": 0.005600000000000001, - "prompt_tokens": 120, - "spend": 0.014000000000000002 - }, - "claude-sonnet-5|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0048000000000000004, - "output_cost": 0.0030000000000000005, - "prompt_tokens": 80, - "spend": 0.007800000000000001 - }, - "claude-sonnet-5|tool_call": { - "completion_tokens": 40, - "input_cost": 0.007200000000000001, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 120, - "spend": 0.012 - }, - "claude-sonnet-5|web_search": { - "completion_tokens": 30, - "input_cost": 0.006000000000000001, - "output_cost": 0.0036000000000000003, - "prompt_tokens": 100, - "spend": 0.0696 - }, - "fireworks_ai/deepseek-v4p1-flash|all_components_fireworks": { - "completion_tokens": 25, - "input_cost": 0.011760000000000001, - "output_cost": 0.007000000000000001, - "prompt_tokens": 120, - "spend": 0.018760000000000002 - }, - "fireworks_ai/deepseek-v4p1-flash|audio": { - "completion_tokens": 45, - "input_cost": 0.030500000000000003, - "output_cost": 0.019950000000000002, - "prompt_tokens": 125, - "spend": 0.05045000000000001 - }, - "fireworks_ai/deepseek-v4p1-flash|basic": { - "completion_tokens": 40, - "input_cost": 0.016800000000000002, - "output_cost": 0.011200000000000002, - "prompt_tokens": 120, - "spend": 0.028000000000000004 - }, - "fireworks_ai/deepseek-v4p1-flash|cache_read": { - "completion_tokens": 30, - "input_cost": 0.014700000000000001, - "output_cost": 0.008400000000000001, - "prompt_tokens": 150, - "spend": 0.023100000000000002 - }, - "fireworks_ai/deepseek-v4p1-flash|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.0368, - "output_cost": 0.008400000000000001, - "prompt_tokens": 150, - "spend": 0.045200000000000004 - }, - "fireworks_ai/deepseek-v4p1-flash|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0324, - "output_cost": 0.008400000000000001, - "prompt_tokens": 150, - "spend": 0.0408 - }, - "fireworks_ai/deepseek-v4p1-flash|reasoning": { - "completion_tokens": 100, - "input_cost": 0.014000000000000002, - "output_cost": 0.0469, - "prompt_tokens": 100, - "spend": 0.060899999999999996 - }, - "fireworks_ai/deepseek-v4p1-flash|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.014400000000000001, - "output_cost": 0.009600000000000001, - "prompt_tokens": 120, - "spend": 0.024 - }, - "fireworks_ai/deepseek-v4p1-flash|stream": { - "completion_tokens": 40, - "input_cost": 0.016800000000000002, - "output_cost": 0.011200000000000002, - "prompt_tokens": 120, - "spend": 0.028000000000000004 - }, - "fireworks_ai/deepseek-v4p1-flash|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.014400000000000001, - "output_cost": 0.009600000000000001, - "prompt_tokens": 120, - "spend": 0.024 - }, - "fireworks_ai/deepseek-v4p1-flash|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.011200000000000002, - "output_cost": 0.007000000000000001, - "prompt_tokens": 80, - "spend": 0.0182 - }, - "fireworks_ai/deepseek-v4p1-flash|tool_call": { - "completion_tokens": 40, - "input_cost": 0.016800000000000002, - "output_cost": 0.011200000000000002, - "prompt_tokens": 120, - "spend": 0.028000000000000004 - }, - "fireworks_ai/deepseek-v4p1-flash|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.014000000000000002, - "output_cost": 0.008400000000000001, - "prompt_tokens": 100, - "spend": 0.04240000000000001 - }, - "fireworks_ai/kimi-k3|all_components_fireworks": { - "completion_tokens": 25, - "input_cost": 0.01008, - "output_cost": 0.006000000000000001, - "prompt_tokens": 120, - "spend": 0.01608 - }, - "fireworks_ai/kimi-k3|audio": { - "completion_tokens": 45, - "input_cost": 0.028500000000000004, - "output_cost": 0.01875, - "prompt_tokens": 125, - "spend": 0.04725 - }, - "fireworks_ai/kimi-k3|basic": { - "completion_tokens": 40, - "input_cost": 0.014400000000000001, - "output_cost": 0.009600000000000001, - "prompt_tokens": 120, - "spend": 0.024 - }, - "fireworks_ai/kimi-k3|cache_read": { - "completion_tokens": 30, - "input_cost": 0.012600000000000002, - "output_cost": 0.007200000000000001, - "prompt_tokens": 150, - "spend": 0.0198 - }, - "fireworks_ai/kimi-k3|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.035, - "output_cost": 0.007200000000000001, - "prompt_tokens": 150, - "spend": 0.0422 - }, - "fireworks_ai/kimi-k3|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.030600000000000002, - "output_cost": 0.007200000000000001, - "prompt_tokens": 150, - "spend": 0.0378 - }, - "fireworks_ai/kimi-k3|reasoning": { - "completion_tokens": 100, - "input_cost": 0.012000000000000002, - "output_cost": 0.0457, - "prompt_tokens": 100, - "spend": 0.0577 - }, - "fireworks_ai/kimi-k3|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.015600000000000003, - "output_cost": 0.010400000000000001, - "prompt_tokens": 120, - "spend": 0.026000000000000002 - }, - "fireworks_ai/kimi-k3|stream": { - "completion_tokens": 40, - "input_cost": 0.014400000000000001, - "output_cost": 0.009600000000000001, - "prompt_tokens": 120, - "spend": 0.024 - }, - "fireworks_ai/kimi-k3|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.015600000000000003, - "output_cost": 0.010400000000000001, - "prompt_tokens": 120, - "spend": 0.026000000000000002 - }, - "fireworks_ai/kimi-k3|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.009600000000000001, - "output_cost": 0.006000000000000001, - "prompt_tokens": 80, - "spend": 0.015600000000000003 - }, - "fireworks_ai/kimi-k3|tool_call": { - "completion_tokens": 40, - "input_cost": 0.014400000000000001, - "output_cost": 0.009600000000000001, - "prompt_tokens": 120, - "spend": 0.024 - }, - "fireworks_ai/kimi-k3|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.012000000000000002, - "output_cost": 0.007200000000000001, - "prompt_tokens": 100, - "spend": 0.0392 - }, - "fireworks_ai/qwen3p8-max|all_components_fireworks": { - "completion_tokens": 25, - "input_cost": 0.010920000000000001, - "output_cost": 0.006500000000000001, - "prompt_tokens": 120, - "spend": 0.01742 - }, - "fireworks_ai/qwen3p8-max|audio": { - "completion_tokens": 45, - "input_cost": 0.029500000000000002, - "output_cost": 0.01935, - "prompt_tokens": 125, - "spend": 0.048850000000000005 - }, - "fireworks_ai/qwen3p8-max|basic": { - "completion_tokens": 40, - "input_cost": 0.015600000000000003, - "output_cost": 0.010400000000000001, - "prompt_tokens": 120, - "spend": 0.026000000000000002 - }, - "fireworks_ai/qwen3p8-max|cache_read": { - "completion_tokens": 30, - "input_cost": 0.01365, - "output_cost": 0.007800000000000001, - "prompt_tokens": 150, - "spend": 0.021450000000000004 - }, - "fireworks_ai/qwen3p8-max|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.0359, - "output_cost": 0.007800000000000001, - "prompt_tokens": 150, - "spend": 0.0437 - }, - "fireworks_ai/qwen3p8-max|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0315, - "output_cost": 0.007800000000000001, - "prompt_tokens": 150, - "spend": 0.0393 - }, - "fireworks_ai/qwen3p8-max|reasoning": { - "completion_tokens": 100, - "input_cost": 0.013000000000000001, - "output_cost": 0.0463, - "prompt_tokens": 100, - "spend": 0.059300000000000005 - }, - "fireworks_ai/qwen3p8-max|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.016800000000000002, - "output_cost": 0.011200000000000002, - "prompt_tokens": 120, - "spend": 0.028000000000000004 - }, - "fireworks_ai/qwen3p8-max|stream": { - "completion_tokens": 40, - "input_cost": 0.015600000000000003, - "output_cost": 0.010400000000000001, - "prompt_tokens": 120, - "spend": 0.026000000000000002 - }, - "fireworks_ai/qwen3p8-max|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.016800000000000002, - "output_cost": 0.011200000000000002, - "prompt_tokens": 120, - "spend": 0.028000000000000004 - }, - "fireworks_ai/qwen3p8-max|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.010400000000000001, - "output_cost": 0.006500000000000001, - "prompt_tokens": 80, - "spend": 0.016900000000000002 - }, - "fireworks_ai/qwen3p8-max|tool_call": { - "completion_tokens": 40, - "input_cost": 0.015600000000000003, - "output_cost": 0.010400000000000001, - "prompt_tokens": 120, - "spend": 0.026000000000000002 - }, - "fireworks_ai/qwen3p8-max|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.013000000000000001, - "output_cost": 0.007800000000000001, - "prompt_tokens": 100, - "spend": 0.0408 - }, - "gemini-3.1-pro-preview|all_components_gemini": { - "completion_tokens": 43, - "input_cost": 0.023940000000000003, - "output_cost": 0.030660000000000003, - "prompt_tokens": 125, - "spend": 0.05460000000000001 - }, - "gemini-3.1-pro-preview|audio": { - "completion_tokens": 45, - "input_cost": 0.052500000000000005, - "output_cost": 0.03465, - "prompt_tokens": 125, - "spend": 0.08715 - }, - "gemini-3.1-pro-preview|basic": { - "completion_tokens": 40, - "input_cost": 0.0252, - "output_cost": 0.016800000000000002, - "prompt_tokens": 120, - "spend": 0.042 - }, - "gemini-3.1-pro-preview|cache_read": { - "completion_tokens": 30, - "input_cost": 0.02205, - "output_cost": 0.0126, - "prompt_tokens": 150, - "spend": 0.03465 - }, - "gemini-3.1-pro-preview|prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.2, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.2 - }, - "gemini-3.1-pro-preview|reasoning": { - "completion_tokens": 100, - "input_cost": 0.021, - "output_cost": 0.0861, - "prompt_tokens": 100, - "spend": 0.1071 - }, - "gemini-3.1-pro-preview|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.024, - "output_cost": 0.016, - "prompt_tokens": 120, - "spend": 0.04 - }, - "gemini-3.1-pro-preview|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0378, - "output_cost": 0.020999999999999998, - "prompt_tokens": 120, - "spend": 0.0588 - }, - "gemini-3.1-pro-preview|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.04284, - "output_cost": 0.02268, - "prompt_tokens": 120, - "spend": 0.06552 - }, - "gemini-3.1-pro-preview|stream": { - "completion_tokens": 40, - "input_cost": 0.0252, - "output_cost": 0.016800000000000002, - "prompt_tokens": 120, - "spend": 0.042 - }, - "gemini-3.1-pro-preview|stream_prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.2, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.2 - }, - "gemini-3.1-pro-preview|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.024, - "output_cost": 0.016, - "prompt_tokens": 120, - "spend": 0.04 - }, - "gemini-3.1-pro-preview|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.016800000000000002, - "output_cost": 0.0105, - "prompt_tokens": 80, - "spend": 0.027300000000000005 - }, - "gemini-3.1-pro-preview|tiered": { - "completion_tokens": 30, - "input_cost": 336.00168, - "output_cost": 0.0567, - "prompt_tokens": 200001, - "spend": 336.05838 - }, - "gemini-3.1-pro-preview|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0252, - "output_cost": 0.016800000000000002, - "prompt_tokens": 120, - "spend": 0.042 - }, - "gemini-3.1-pro-preview|web_search": { - "completion_tokens": 30, - "input_cost": 0.021, - "output_cost": 0.0126, - "prompt_tokens": 100, - "spend": 0.0936 - }, - "gemini-3.8-flash|all_components_gemini": { - "completion_tokens": 43, - "input_cost": 0.022799999999999997, - "output_cost": 0.0292, - "prompt_tokens": 125, - "spend": 0.052 - }, - "gemini-3.8-flash|audio": { - "completion_tokens": 45, - "input_cost": 0.05, - "output_cost": 0.033, - "prompt_tokens": 125, - "spend": 0.083 - }, - "gemini-3.8-flash|basic": { - "completion_tokens": 40, - "input_cost": 0.024, - "output_cost": 0.016, - "prompt_tokens": 120, - "spend": 0.04 - }, - "gemini-3.8-flash|cache_read": { - "completion_tokens": 30, - "input_cost": 0.021, - "output_cost": 0.012, - "prompt_tokens": 150, - "spend": 0.033 - }, - "gemini-3.8-flash|prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.21000000000000002, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.21000000000000002 - }, - "gemini-3.8-flash|reasoning": { - "completion_tokens": 100, - "input_cost": 0.02, - "output_cost": 0.082, - "prompt_tokens": 100, - "spend": 0.10200000000000001 - }, - "gemini-3.8-flash|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0252, - "output_cost": 0.016800000000000002, - "prompt_tokens": 120, - "spend": 0.042 - }, - "gemini-3.8-flash|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.036, - "output_cost": 0.02, - "prompt_tokens": 120, - "spend": 0.055999999999999994 - }, - "gemini-3.8-flash|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.0408, - "output_cost": 0.0216, - "prompt_tokens": 120, - "spend": 0.062400000000000004 - }, - "gemini-3.8-flash|stream": { - "completion_tokens": 40, - "input_cost": 0.024, - "output_cost": 0.016, - "prompt_tokens": 120, - "spend": 0.04 - }, - "gemini-3.8-flash|stream_prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.21000000000000002, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.21000000000000002 - }, - "gemini-3.8-flash|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0252, - "output_cost": 0.016800000000000002, - "prompt_tokens": 120, - "spend": 0.042 - }, - "gemini-3.8-flash|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.016, - "output_cost": 0.01, - "prompt_tokens": 80, - "spend": 0.026000000000000002 - }, - "gemini-3.8-flash|tiered": { - "completion_tokens": 30, - "input_cost": 320.0016, - "output_cost": 0.054, - "prompt_tokens": 200001, - "spend": 320.05559999999997 - }, - "gemini-3.8-flash|tool_call": { - "completion_tokens": 40, - "input_cost": 0.024, - "output_cost": 0.016, - "prompt_tokens": 120, - "spend": 0.04 - }, - "gemini-3.8-flash|web_search": { - "completion_tokens": 30, - "input_cost": 0.02, - "output_cost": 0.012, - "prompt_tokens": 100, - "spend": 0.092 - }, - "gemini/gemini-3.1-pro-preview|all_components_gemini": { - "completion_tokens": 43, - "input_cost": 0.010260000000000002, - "output_cost": 0.01314, - "prompt_tokens": 125, - "spend": 0.023400000000000004 - }, - "gemini/gemini-3.1-pro-preview|audio": { - "completion_tokens": 45, - "input_cost": 0.0225, - "output_cost": 0.014849999999999999, - "prompt_tokens": 125, - "spend": 0.037349999999999994 - }, - "gemini/gemini-3.1-pro-preview|basic": { - "completion_tokens": 40, - "input_cost": 0.0108, - "output_cost": 0.007200000000000001, - "prompt_tokens": 120, - "spend": 0.018000000000000002 - }, - "gemini/gemini-3.1-pro-preview|cache_read": { - "completion_tokens": 30, - "input_cost": 0.009450000000000002, - "output_cost": 0.0054, - "prompt_tokens": 150, - "spend": 0.014850000000000002 - }, - "gemini/gemini-3.1-pro-preview|prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.08, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.08 - }, - "gemini/gemini-3.1-pro-preview|reasoning": { - "completion_tokens": 100, - "input_cost": 0.009000000000000001, - "output_cost": 0.0369, - "prompt_tokens": 100, - "spend": 0.0459 - }, - "gemini/gemini-3.1-pro-preview|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.009600000000000001, - "output_cost": 0.0064, - "prompt_tokens": 120, - "spend": 0.016 - }, - "gemini/gemini-3.1-pro-preview|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0162, - "output_cost": 0.009000000000000001, - "prompt_tokens": 120, - "spend": 0.0252 - }, - "gemini/gemini-3.1-pro-preview|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.01836, - "output_cost": 0.00972, - "prompt_tokens": 120, - "spend": 0.02808 - }, - "gemini/gemini-3.1-pro-preview|stream": { - "completion_tokens": 40, - "input_cost": 0.0108, - "output_cost": 0.007200000000000001, - "prompt_tokens": 120, - "spend": 0.018000000000000002 - }, - "gemini/gemini-3.1-pro-preview|stream_prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.08, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.08 - }, - "gemini/gemini-3.1-pro-preview|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.009600000000000001, - "output_cost": 0.0064, - "prompt_tokens": 120, - "spend": 0.016 - }, - "gemini/gemini-3.1-pro-preview|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.007200000000000001, - "output_cost": 0.0045000000000000005, - "prompt_tokens": 80, - "spend": 0.011700000000000002 - }, - "gemini/gemini-3.1-pro-preview|tiered": { - "completion_tokens": 30, - "input_cost": 144.00072, - "output_cost": 0.024300000000000002, - "prompt_tokens": 200001, - "spend": 144.02502 - }, - "gemini/gemini-3.1-pro-preview|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0108, - "output_cost": 0.007200000000000001, - "prompt_tokens": 120, - "spend": 0.018000000000000002 - }, - "gemini/gemini-3.1-pro-preview|web_search": { - "completion_tokens": 30, - "input_cost": 0.009000000000000001, - "output_cost": 0.0054, - "prompt_tokens": 100, - "spend": 0.0744 - }, - "gemini/gemini-3.8-flash|all_components_gemini": { - "completion_tokens": 43, - "input_cost": 0.00912, - "output_cost": 0.01168, - "prompt_tokens": 125, - "spend": 0.0208 - }, - "gemini/gemini-3.8-flash|audio": { - "completion_tokens": 45, - "input_cost": 0.02, - "output_cost": 0.0132, - "prompt_tokens": 125, - "spend": 0.0332 - }, - "gemini/gemini-3.8-flash|basic": { - "completion_tokens": 40, - "input_cost": 0.009600000000000001, - "output_cost": 0.0064, - "prompt_tokens": 120, - "spend": 0.016 - }, - "gemini/gemini-3.8-flash|cache_read": { - "completion_tokens": 30, - "input_cost": 0.0084, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 150, - "spend": 0.0132 - }, - "gemini/gemini-3.8-flash|prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.09000000000000001, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.09000000000000001 - }, - "gemini/gemini-3.8-flash|reasoning": { - "completion_tokens": 100, - "input_cost": 0.008, - "output_cost": 0.0328, - "prompt_tokens": 100, - "spend": 0.0408 - }, - "gemini/gemini-3.8-flash|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0108, - "output_cost": 0.007200000000000001, - "prompt_tokens": 120, - "spend": 0.018000000000000002 - }, - "gemini/gemini-3.8-flash|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0144, - "output_cost": 0.008, - "prompt_tokens": 120, - "spend": 0.0224 - }, - "gemini/gemini-3.8-flash|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.01632, - "output_cost": 0.00864, - "prompt_tokens": 120, - "spend": 0.024960000000000003 - }, - "gemini/gemini-3.8-flash|stream": { - "completion_tokens": 40, - "input_cost": 0.009600000000000001, - "output_cost": 0.0064, - "prompt_tokens": 120, - "spend": 0.016 - }, - "gemini/gemini-3.8-flash|stream_prompt_blocked": { - "completion_tokens": 0, - "input_cost": 0.09000000000000001, - "output_cost": 0.0, - "prompt_tokens": 1000, - "spend": 0.09000000000000001 - }, - "gemini/gemini-3.8-flash|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0108, - "output_cost": 0.007200000000000001, - "prompt_tokens": 120, - "spend": 0.018000000000000002 - }, - "gemini/gemini-3.8-flash|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0064, - "output_cost": 0.004, - "prompt_tokens": 80, - "spend": 0.0104 - }, - "gemini/gemini-3.8-flash|tiered": { - "completion_tokens": 30, - "input_cost": 128.00064, - "output_cost": 0.0216, - "prompt_tokens": 200001, - "spend": 128.02224 - }, - "gemini/gemini-3.8-flash|tool_call": { - "completion_tokens": 40, - "input_cost": 0.009600000000000001, - "output_cost": 0.0064, - "prompt_tokens": 120, - "spend": 0.016 - }, - "gemini/gemini-3.8-flash|web_search": { - "completion_tokens": 30, - "input_cost": 0.008, - "output_cost": 0.0048000000000000004, - "prompt_tokens": 100, - "spend": 0.0728 - }, - "gpt-5.3-codex|all_components_responses": { - "completion_tokens": 40, - "input_cost": 0.00252, - "output_cost": 0.0037500000000000007, - "prompt_tokens": 120, - "spend": 0.006270000000000001 - }, - "gpt-5.3-codex|basic": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.3-codex|cache_read": { - "completion_tokens": 30, - "input_cost": 0.0031500000000000005, - "output_cost": 0.0018000000000000002, - "prompt_tokens": 150, - "spend": 0.00495 - }, - "gpt-5.3-codex|reasoning": { - "completion_tokens": 100, - "input_cost": 0.0030000000000000005, - "output_cost": 0.0123, - "prompt_tokens": 100, - "spend": 0.015300000000000001 - }, - "gpt-5.3-codex|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.3-codex|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0054, - "output_cost": 0.003, - "prompt_tokens": 120, - "spend": 0.008400000000000001 - }, - "gpt-5.3-codex|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.00612, - "output_cost": 0.00324, - "prompt_tokens": 120, - "spend": 0.00936 - }, - "gpt-5.3-codex|stream": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.3-codex|stream_incomplete": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.3-codex|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.3-codex|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0015000000000000002, - "prompt_tokens": 80, - "spend": 0.0039000000000000007 - }, - "gpt-5.3-codex|stream_unvalidated": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.3-codex|tiered": { - "completion_tokens": 30, - "input_cost": 48.000240000000005, - "output_cost": 0.0081, - "prompt_tokens": 200001, - "spend": 48.008340000000004 - }, - "gpt-5.3-codex|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.3-codex|web_search": { - "completion_tokens": 30, - "input_cost": 0.0030000000000000005, - "output_cost": 0.0018000000000000002, - "prompt_tokens": 100, - "spend": 0.0648 - }, - "gpt-5.4-mini|all_components_chat": { - "completion_tokens": 43, - "input_cost": 0.00856, - "output_cost": 0.00584, - "prompt_tokens": 155, - "spend": 0.0144 - }, - "gpt-5.4-mini|audio": { - "completion_tokens": 45, - "input_cost": 0.01, - "output_cost": 0.0066, - "prompt_tokens": 125, - "spend": 0.0166 - }, - "gpt-5.4-mini|basic": { - "completion_tokens": 40, - "input_cost": 0.0048000000000000004, - "output_cost": 0.0032, - "prompt_tokens": 120, - "spend": 0.008 - }, - "gpt-5.4-mini|cache_read": { - "completion_tokens": 30, - "input_cost": 0.0042, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 150, - "spend": 0.0066 - }, - "gpt-5.4-mini|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.012400000000000001, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 150, - "spend": 0.0148 - }, - "gpt-5.4-mini|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0108, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 150, - "spend": 0.0132 - }, - "gpt-5.4-mini|reasoning": { - "completion_tokens": 100, - "input_cost": 0.004, - "output_cost": 0.0164, - "prompt_tokens": 100, - "spend": 0.0204 - }, - "gpt-5.4-mini|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0012000000000000001, - "output_cost": 0.0008, - "prompt_tokens": 120, - "spend": 0.002 - }, - "gpt-5.4-mini|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0072, - "output_cost": 0.004, - "prompt_tokens": 120, - "spend": 0.0112 - }, - "gpt-5.4-mini|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.00816, - "output_cost": 0.00432, - "prompt_tokens": 120, - "spend": 0.012480000000000002 - }, - "gpt-5.4-mini|stream": { - "completion_tokens": 40, - "input_cost": 0.0048000000000000004, - "output_cost": 0.0032, - "prompt_tokens": 120, - "spend": 0.008 - }, - "gpt-5.4-mini|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0012000000000000001, - "output_cost": 0.0008, - "prompt_tokens": 120, - "spend": 0.002 - }, - "gpt-5.4-mini|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0032, - "output_cost": 0.002, - "prompt_tokens": 80, - "spend": 0.0052 - }, - "gpt-5.4-mini|tiered": { - "completion_tokens": 30, - "input_cost": 64.00032, - "output_cost": 0.0108, - "prompt_tokens": 200001, - "spend": 64.01112 - }, - "gpt-5.4-mini|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0048000000000000004, - "output_cost": 0.0032, - "prompt_tokens": 120, - "spend": 0.008 - }, - "gpt-5.4-mini|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.004, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 100, - "spend": 0.0264 - }, - "gpt-5.5-pro|all_components_responses": { - "completion_tokens": 40, - "input_cost": 0.00168, - "output_cost": 0.0025, - "prompt_tokens": 120, - "spend": 0.00418 - }, - "gpt-5.5-pro|basic": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.5-pro|cache_read": { - "completion_tokens": 30, - "input_cost": 0.0021, - "output_cost": 0.0012000000000000001, - "prompt_tokens": 150, - "spend": 0.0033 - }, - "gpt-5.5-pro|reasoning": { - "completion_tokens": 100, - "input_cost": 0.002, - "output_cost": 0.0082, - "prompt_tokens": 100, - "spend": 0.0102 - }, - "gpt-5.5-pro|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.5-pro|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0036, - "output_cost": 0.002, - "prompt_tokens": 120, - "spend": 0.0056 - }, - "gpt-5.5-pro|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.00408, - "output_cost": 0.00216, - "prompt_tokens": 120, - "spend": 0.006240000000000001 - }, - "gpt-5.5-pro|stream": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.5-pro|stream_incomplete": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.5-pro|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0036000000000000003, - "output_cost": 0.0024000000000000002, - "prompt_tokens": 120, - "spend": 0.006 - }, - "gpt-5.5-pro|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0016, - "output_cost": 0.001, - "prompt_tokens": 80, - "spend": 0.0026 - }, - "gpt-5.5-pro|stream_unvalidated": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.5-pro|tiered": { - "completion_tokens": 30, - "input_cost": 32.00016, - "output_cost": 0.0054, - "prompt_tokens": 200001, - "spend": 32.00556 - }, - "gpt-5.5-pro|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0024000000000000002, - "output_cost": 0.0016, - "prompt_tokens": 120, - "spend": 0.004 - }, - "gpt-5.5-pro|web_search": { - "completion_tokens": 30, - "input_cost": 0.002, - "output_cost": 0.0012000000000000001, - "prompt_tokens": 100, - "spend": 0.06319999999999999 - }, - "gpt-5.6|all_components_chat": { - "completion_tokens": 43, - "input_cost": 0.00214, - "output_cost": 0.00146, - "prompt_tokens": 155, - "spend": 0.0036 - }, - "gpt-5.6|audio": { - "completion_tokens": 45, - "input_cost": 0.0025, - "output_cost": 0.00165, - "prompt_tokens": 125, - "spend": 0.00415 - }, - "gpt-5.6|basic": { - "completion_tokens": 40, - "input_cost": 0.0012000000000000001, - "output_cost": 0.0008, - "prompt_tokens": 120, - "spend": 0.002 - }, - "gpt-5.6|cache_read": { - "completion_tokens": 30, - "input_cost": 0.00105, - "output_cost": 0.0006000000000000001, - "prompt_tokens": 150, - "spend": 0.00165 - }, - "gpt-5.6|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.0031000000000000003, - "output_cost": 0.0006000000000000001, - "prompt_tokens": 150, - "spend": 0.0037 - }, - "gpt-5.6|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.0027, - "output_cost": 0.0006000000000000001, - "prompt_tokens": 150, - "spend": 0.0033 - }, - "gpt-5.6|reasoning": { - "completion_tokens": 100, - "input_cost": 0.001, - "output_cost": 0.0041, - "prompt_tokens": 100, - "spend": 0.0051 - }, - "gpt-5.6|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0048000000000000004, - "output_cost": 0.0032, - "prompt_tokens": 120, - "spend": 0.008 - }, - "gpt-5.6|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.0018, - "output_cost": 0.001, - "prompt_tokens": 120, - "spend": 0.0028 - }, - "gpt-5.6|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.00204, - "output_cost": 0.00108, - "prompt_tokens": 120, - "spend": 0.0031200000000000004 - }, - "gpt-5.6|stream": { - "completion_tokens": 40, - "input_cost": 0.0012000000000000001, - "output_cost": 0.0008, - "prompt_tokens": 120, - "spend": 0.002 - }, - "gpt-5.6|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0048000000000000004, - "output_cost": 0.0032, - "prompt_tokens": 120, - "spend": 0.008 - }, - "gpt-5.6|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0008, - "output_cost": 0.0005, - "prompt_tokens": 80, - "spend": 0.0013 - }, - "gpt-5.6|tiered": { - "completion_tokens": 30, - "input_cost": 16.00008, - "output_cost": 0.0027, - "prompt_tokens": 200001, - "spend": 16.00278 - }, - "gpt-5.6|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0012000000000000001, - "output_cost": 0.0008, - "prompt_tokens": 120, - "spend": 0.002 - }, - "gpt-5.6|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.001, - "output_cost": 0.0006000000000000001, - "prompt_tokens": 100, - "spend": 0.0216 - }, - "meta.llama4-maverick-17b-instruct-v1:0|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.0285, - "output_cost": 0.0095, - "prompt_tokens": 150, - "spend": 0.038 - }, - "meta.llama4-maverick-17b-instruct-v1:0|basic": { - "completion_tokens": 40, - "input_cost": 0.0228, - "output_cost": 0.015200000000000002, - "prompt_tokens": 120, - "spend": 0.038000000000000006 - }, - "meta.llama4-maverick-17b-instruct-v1:0|stream": { - "completion_tokens": 40, - "input_cost": 0.0228, - "output_cost": 0.015200000000000002, - "prompt_tokens": 120, - "spend": 0.038000000000000006 - }, - "meta.llama4-maverick-17b-instruct-v1:0|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.015200000000000002, - "output_cost": 0.0095, - "prompt_tokens": 80, - "spend": 0.0247 - }, - "meta.llama4-maverick-17b-instruct-v1:0|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0228, - "output_cost": 0.015200000000000002, - "prompt_tokens": 120, - "spend": 0.038000000000000006 - }, - "together_ai/moonshotai/Kimi-K3|all_components_chat": { - "completion_tokens": 43, - "input_cost": 0.0214, - "output_cost": 0.0146, - "prompt_tokens": 155, - "spend": 0.036 - }, - "together_ai/moonshotai/Kimi-K3|audio": { - "completion_tokens": 45, - "input_cost": 0.025, - "output_cost": 0.0165, - "prompt_tokens": 125, - "spend": 0.0415 - }, - "together_ai/moonshotai/Kimi-K3|basic": { - "completion_tokens": 40, - "input_cost": 0.012, - "output_cost": 0.008, - "prompt_tokens": 120, - "spend": 0.02 - }, - "together_ai/moonshotai/Kimi-K3|cache_read": { - "completion_tokens": 30, - "input_cost": 0.0105, - "output_cost": 0.006, - "prompt_tokens": 150, - "spend": 0.0165 - }, - "together_ai/moonshotai/Kimi-K3|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.031, - "output_cost": 0.006, - "prompt_tokens": 150, - "spend": 0.037 - }, - "together_ai/moonshotai/Kimi-K3|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.027000000000000003, - "output_cost": 0.006, - "prompt_tokens": 150, - "spend": 0.033 - }, - "together_ai/moonshotai/Kimi-K3|reasoning": { - "completion_tokens": 100, - "input_cost": 0.01, - "output_cost": 0.041, - "prompt_tokens": 100, - "spend": 0.051000000000000004 - }, - "together_ai/moonshotai/Kimi-K3|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0132, - "output_cost": 0.0088, - "prompt_tokens": 120, - "spend": 0.022 - }, - "together_ai/moonshotai/Kimi-K3|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.018000000000000002, - "output_cost": 0.01, - "prompt_tokens": 120, - "spend": 0.028000000000000004 - }, - "together_ai/moonshotai/Kimi-K3|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.0204, - "output_cost": 0.0108, - "prompt_tokens": 120, - "spend": 0.031200000000000002 - }, - "together_ai/moonshotai/Kimi-K3|stream": { - "completion_tokens": 40, - "input_cost": 0.012, - "output_cost": 0.008, - "prompt_tokens": 120, - "spend": 0.02 - }, - "together_ai/moonshotai/Kimi-K3|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.0132, - "output_cost": 0.0088, - "prompt_tokens": 120, - "spend": 0.022 - }, - "together_ai/moonshotai/Kimi-K3|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.008, - "output_cost": 0.005, - "prompt_tokens": 80, - "spend": 0.013000000000000001 - }, - "together_ai/moonshotai/Kimi-K3|tiered": { - "completion_tokens": 30, - "input_cost": 160.0008, - "output_cost": 0.027000000000000003, - "prompt_tokens": 200001, - "spend": 160.02779999999998 - }, - "together_ai/moonshotai/Kimi-K3|tool_call": { - "completion_tokens": 40, - "input_cost": 0.012, - "output_cost": 0.008, - "prompt_tokens": 120, - "spend": 0.02 - }, - "together_ai/moonshotai/Kimi-K3|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.01, - "output_cost": 0.006, - "prompt_tokens": 100, - "spend": 0.036000000000000004 - }, - "together_ai/zai-org/GLM-5.3|all_components_chat": { - "completion_tokens": 43, - "input_cost": 0.023540000000000002, - "output_cost": 0.01606, - "prompt_tokens": 155, - "spend": 0.0396 - }, - "together_ai/zai-org/GLM-5.3|audio": { - "completion_tokens": 45, - "input_cost": 0.027500000000000004, - "output_cost": 0.01815, - "prompt_tokens": 125, - "spend": 0.04565 - }, - "together_ai/zai-org/GLM-5.3|basic": { - "completion_tokens": 40, - "input_cost": 0.0132, - "output_cost": 0.0088, - "prompt_tokens": 120, - "spend": 0.022 - }, - "together_ai/zai-org/GLM-5.3|cache_read": { - "completion_tokens": 30, - "input_cost": 0.011550000000000001, - "output_cost": 0.0066, - "prompt_tokens": 150, - "spend": 0.01815 - }, - "together_ai/zai-org/GLM-5.3|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.034100000000000005, - "output_cost": 0.0066, - "prompt_tokens": 150, - "spend": 0.04070000000000001 - }, - "together_ai/zai-org/GLM-5.3|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.029699999999999997, - "output_cost": 0.0066, - "prompt_tokens": 150, - "spend": 0.0363 - }, - "together_ai/zai-org/GLM-5.3|reasoning": { - "completion_tokens": 100, - "input_cost": 0.011000000000000001, - "output_cost": 0.0451, - "prompt_tokens": 100, - "spend": 0.056100000000000004 - }, - "together_ai/zai-org/GLM-5.3|response_model_override": { - "completion_tokens": 40, - "input_cost": 0.012, - "output_cost": 0.008, - "prompt_tokens": 120, - "spend": 0.02 - }, - "together_ai/zai-org/GLM-5.3|service_tier_flex": { - "completion_tokens": 40, - "input_cost": 0.019799999999999998, - "output_cost": 0.011000000000000001, - "prompt_tokens": 120, - "spend": 0.0308 - }, - "together_ai/zai-org/GLM-5.3|service_tier_priority": { - "completion_tokens": 40, - "input_cost": 0.022439999999999998, - "output_cost": 0.01188, - "prompt_tokens": 120, - "spend": 0.034319999999999996 - }, - "together_ai/zai-org/GLM-5.3|stream": { - "completion_tokens": 40, - "input_cost": 0.0132, - "output_cost": 0.0088, - "prompt_tokens": 120, - "spend": 0.022 - }, - "together_ai/zai-org/GLM-5.3|stream_response_model_override": { - "completion_tokens": 40, - "input_cost": 0.012, - "output_cost": 0.008, - "prompt_tokens": 120, - "spend": 0.02 - }, - "together_ai/zai-org/GLM-5.3|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.0088, - "output_cost": 0.0055000000000000005, - "prompt_tokens": 80, - "spend": 0.0143 - }, - "together_ai/zai-org/GLM-5.3|tiered": { - "completion_tokens": 30, - "input_cost": 176.00088, - "output_cost": 0.0297, - "prompt_tokens": 200001, - "spend": 176.03058 - }, - "together_ai/zai-org/GLM-5.3|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0132, - "output_cost": 0.0088, - "prompt_tokens": 120, - "spend": 0.022 - }, - "together_ai/zai-org/GLM-5.3|web_search_single": { - "completion_tokens": 30, - "input_cost": 0.011000000000000001, - "output_cost": 0.0066, - "prompt_tokens": 100, - "spend": 0.0376 - }, - "us.anthropic.claude-opus-5-v1:0|all_components_anthropic": { - "completion_tokens": 25, - "input_cost": 0.033120000000000004, - "output_cost": 0.009000000000000001, - "prompt_tokens": 150, - "spend": 0.042120000000000005 - }, - "us.anthropic.claude-opus-5-v1:0|basic": { - "completion_tokens": 40, - "input_cost": 0.0216, - "output_cost": 0.014400000000000001, - "prompt_tokens": 120, - "spend": 0.036000000000000004 - }, - "us.anthropic.claude-opus-5-v1:0|cache_read": { - "completion_tokens": 30, - "input_cost": 0.018900000000000004, - "output_cost": 0.0108, - "prompt_tokens": 150, - "spend": 0.029700000000000004 - }, - "us.anthropic.claude-opus-5-v1:0|cache_write_1h": { - "completion_tokens": 30, - "input_cost": 0.0558, - "output_cost": 0.0108, - "prompt_tokens": 150, - "spend": 0.0666 - }, - "us.anthropic.claude-opus-5-v1:0|cache_write_5m": { - "completion_tokens": 30, - "input_cost": 0.048600000000000004, - "output_cost": 0.0108, - "prompt_tokens": 150, - "spend": 0.05940000000000001 - }, - "us.anthropic.claude-opus-5-v1:0|stream": { - "completion_tokens": 40, - "input_cost": 0.0216, - "output_cost": 0.014400000000000001, - "prompt_tokens": 120, - "spend": 0.036000000000000004 - }, - "us.anthropic.claude-opus-5-v1:0|stream_tool_call": { - "completion_tokens": 25, - "input_cost": 0.014400000000000001, - "output_cost": 0.009000000000000001, - "prompt_tokens": 80, - "spend": 0.023400000000000004 - }, - "us.anthropic.claude-opus-5-v1:0|tool_call": { - "completion_tokens": 40, - "input_cost": 0.0216, - "output_cost": 0.014400000000000001, - "prompt_tokens": 120, - "spend": 0.036000000000000004 - } -} diff --git a/tests/e2e/cost_calculation/generate_expected.py b/tests/e2e/cost_calculation/generate_expected.py deleted file mode 100644 index 64abdb14c99..00000000000 --- a/tests/e2e/cost_calculation/generate_expected.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Golden generator for the cost suite. Run: - - uv run python tests/e2e/cost_calculation/generate_expected.py - -Loads the derived matrix (models x applicable cases), computes the golden for -each exact-spend cell from the rate arithmetic, and writes ``expected.json`` -with sorted keys. Default behaviour adds missing cells and drops stale cells -but never overwrites an existing cell's values (a reviewed golden is -authoritative); ``--rewrite`` recomputes everything. Prints added/removed/kept -counts. -""" - -from __future__ import annotations - -import json -import sys -from collections.abc import Mapping -from dataclasses import dataclass -from types import MappingProxyType -from typing import Final - -from cost_matrix import ( - EXPECTED_PATH, - FRONTIER_MODELS, - TIER_THRESHOLD_TOKENS, - Case, - CostMapEntry, - ExpectedCell, - FrontierModel, - cases_for, - expected_key, -) -from pydantic import TypeAdapter - - -def _first_present(*rates: float | None) -> float | None: - return next((rate for rate in rates if rate is not None), None) - - -@dataclass(frozen=True, slots=True) -class ExpectedCost: - """The expected bill split the way the spend row's cost_breakdown reports - it: the gross input component (cache reads/writes folded in), the output - component, and the tool-usage component.""" - - input_cost: float - output_cost: float - tool_cost: float - - @property - def total(self) -> float: - return self.input_cost + self.output_cost + self.tool_cost - - -def expected_breakdown(model: FrontierModel, case: Case) -> ExpectedCost: - """Literal arithmetic on the test-map rates over the scripted token counts. - - Input = fresh*in + read*read + 5m*create + 1h*create_1h + audio_in*audio_in; - output = text*out + reasoning*reasoning + audio_out*audio_out; plus the - billed web-search calls at the medium search-context rate. Every billed - token is a token the provider charged for: a component whose entry has no - dedicated rate bills at the ordinary input or output rate, and a present - rate (including an explicit 0.0) is authoritative. When the total prompt - tokens exceed the threshold, input/output rates come from the - ``_above_200k_tokens`` variants; a service tier takes its ``_priority`` or - ``_flex`` variant when the entry carries one, and otherwise bills at the - base rate. - """ - rates: Final[CostMapEntry] = model.override_rates if case.response_model_override else model.rates - u: Final = case.usage - prompt_tokens: Final = ( - u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens - + u.cache_write_1h_tokens + u.audio_input_tokens - ) - tiered: Final = prompt_tokens > TIER_THRESHOLD_TOKENS - in_rate: Final = ( - _first_present( - rates.input_cost_per_token_above_200k_tokens if tiered else None, - rates.input_cost_per_token_priority if case.service_tier == "priority" else None, - rates.input_cost_per_token_flex if case.service_tier == "flex" else None, - rates.input_cost_per_token, - ) - or 0.0 - ) - out_rate: Final = ( - _first_present( - rates.output_cost_per_token_above_200k_tokens if tiered else None, - rates.output_cost_per_token_priority if case.service_tier == "priority" else None, - rates.output_cost_per_token_flex if case.service_tier == "flex" else None, - rates.output_cost_per_token, - ) - or 0.0 - ) - read_rate: Final = _first_present(rates.cache_read_input_token_cost, in_rate) or 0.0 - write_rate: Final = _first_present(rates.cache_creation_input_token_cost, in_rate) or 0.0 - write_1h_rate: Final = ( - _first_present(rates.cache_creation_input_token_cost_above_1hr, write_rate) or 0.0 - ) - audio_in_rate: Final = _first_present(rates.input_cost_per_audio_token, in_rate) or 0.0 - reasoning_rate: Final = _first_present(rates.output_cost_per_reasoning_token, out_rate) or 0.0 - audio_out_rate: Final = _first_present(rates.output_cost_per_audio_token, out_rate) or 0.0 - input_cost: Final = ( - u.fresh_input_tokens * in_rate - + u.cache_read_tokens * read_rate - + u.cache_write_5m_tokens * write_rate - + u.cache_write_1h_tokens * write_1h_rate - + u.audio_input_tokens * audio_in_rate - ) - output_cost: Final = ( - u.output_tokens * out_rate - + u.reasoning_tokens * reasoning_rate - + u.audio_output_tokens * audio_out_rate - ) - search: Final = rates.search_context_cost_per_query - medium_rate: Final = ( - search.search_context_size_medium if search is not None else None - ) - if u.web_search_calls and medium_rate is None: - raise ValueError( - f"{model.map_key}: case {case.name} bills {u.web_search_calls} web-search " - "calls but the entry has no search_context_cost_per_query medium rate" - ) - tool_cost: Final = u.web_search_calls * (medium_rate if medium_rate is not None else 0.0) - return ExpectedCost(input_cost=input_cost, output_cost=output_cost, tool_cost=tool_cost) - - -def expected_token_columns(model: FrontierModel, case: Case) -> tuple[int, int]: - """(prompt_tokens, completion_tokens) the spend row should carry, per the - wire's normalization: Anthropic folds cache read/write into prompt_tokens, - everyone else reports the totals the wire emitted.""" - u: Final = case.usage - if model.wire in ("anthropic_messages", "bedrock_converse"): - return ( - u.fresh_input_tokens + u.cache_read_tokens + u.cache_write_5m_tokens + u.cache_write_1h_tokens, - u.output_tokens, - ) - if model.wire in ("gemini_generate", "vertex_generate"): - return ( - u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens, - u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, - ) - if model.wire == "openai_responses": - return ( - u.fresh_input_tokens + u.cache_read_tokens, - u.output_tokens + u.reasoning_tokens, - ) - return ( - u.fresh_input_tokens - + u.cache_read_tokens - + u.cache_write_5m_tokens - + u.cache_write_1h_tokens - + u.audio_input_tokens, - u.output_tokens + u.reasoning_tokens + u.audio_output_tokens, - ) - - -def _cell(model: FrontierModel, case: Case) -> ExpectedCell: - breakdown: Final = expected_breakdown(model, case) - prompt_tokens, completion_tokens = expected_token_columns(model, case) - return ExpectedCell( - spend=breakdown.total, - input_cost=breakdown.input_cost, - output_cost=breakdown.output_cost, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - ) - - -def _proposed() -> Mapping[str, ExpectedCell]: - return MappingProxyType( - { - expected_key(model, case): _cell(model, case) - for model in FRONTIER_MODELS - for case in cases_for(model) - if case.exact_spend - } - ) - - -def main() -> None: - rewrite: Final = "--rewrite" in sys.argv[1:] - proposed: Final = _proposed() - proposed_values: Final = {key: cell.model_dump() for key, cell in proposed.items()} - existing: Final[Mapping[str, ExpectedCell]] = ( - TypeAdapter(dict[str, ExpectedCell]).validate_python( - json.loads(EXPECTED_PATH.read_text()) - ) - if EXPECTED_PATH.exists() - else {} - ) - merged: Final = { - key: ( - proposed_values[key] - if rewrite or key not in existing - else existing[key].model_dump() - ) - for key in sorted(proposed_values) - } - added: Final = sum(1 for key in proposed_values if key not in existing) - removed: Final = sum(1 for key in existing if key not in proposed_values) - kept: Final = sum(1 for key in proposed_values if key in existing and not rewrite) - rewritten: Final = sum(1 for key in proposed_values if key in existing and rewrite) - EXPECTED_PATH.write_text(json.dumps(merged, indent=2, sort_keys=True) + "\n") - print( # noqa: T201 # CLI summary is the tool output - f"expected.json: {added} added, {removed} removed, {kept} kept, " - f"{rewritten} rewritten ({len(merged)} cells)" - ) - - -if __name__ == "__main__": - main() diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 346a55aa22d..03dab6be5e7 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -1,7 +1,8 @@ """Token-pricing e2e: every (map entry, case) cell derived from cost_map.json x cases.json runs a scripted-usage call through a deployment registered on the cost-map proxy, and the spend row plus response-cost header must equal the -reviewed golden in expected.json verbatim -- no rate arithmetic lives here. +reviewed golden in the case's ``expected`` cell verbatim -- no rate arithmetic +lives here. Nothing here touches a real provider or the bundled cost map: the proxy's upstream is the scripted-provider sidecar and its entire cost map is @@ -15,13 +16,11 @@ from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( - EXPECTED, FRONTIER_MODELS, IMAGE_INPUT_DATA_URL, Case, FrontierModel, cases_for, - expected_key, matrix_data_errors, recount_cost, ) @@ -142,7 +141,7 @@ class TestTokenPricing: cost_rows.assert_total_is_sum_of_components(row) return - golden: Final = EXPECTED[expected_key(model, case)] + golden: Final = case.expected_for(model) if not case.stream: # Streamed responses commit headers before the bill is computed, so From dda77763464406cd262e1950276cdb5a280c216c Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 18 Sep 2026 13:15:29 +0000 Subject: [PATCH 19/30] test(e2e): make cost-calculation cases MECE by rate-key ownership with realistic fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/CLAUDE.md | 2 +- tests/e2e/cost_calculation/cases.json | 3135 ++++++++++++++--- tests/e2e/cost_calculation/conftest.py | 5 + tests/e2e/cost_calculation/cost_matrix.py | 227 +- .../e2e/cost_calculation/scripted_provider.py | 268 +- .../test_token_pricing_e2e.py | 148 +- tests/e2e/cost_map.json | 800 ++--- tests/e2e/models.py | 63 +- 8 files changed, 3694 insertions(+), 954 deletions(-) diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index a3e5696ef9d..54c143c11d9 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` (each exact-spend case carries a literal `expected` cell per map key; `cost_matrix.matrix_data_errors()` runs at collection time so a key absent from the cost map fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml`, Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in +- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` (each `pricing` case owns (model, cost key) pairs via `owns`/`fallback_for` so every rate key present on each map entry has exactly one owning case, and each carries a literal `expected` cell per map key; `transport` cases list `models` and exercise token counting only; `cost_matrix.matrix_data_errors()` runs at collection time so a key absent from the cost map, an unowned or double-owned (model, rate key) pair, an `owns` key absent on all of the case's models, or a `fallback_for` key present on a case model fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml` with `MODEL_COST_MAP_MIN_MODEL_COUNT=1` and `MODEL_COST_MAP_MAX_SHRINK_RATIO=0` (the 21-entry test map trips the fetched-cost-map integrity check at the defaults), Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` diff --git a/tests/e2e/cost_calculation/cases.json b/tests/e2e/cost_calculation/cases.json index cda7bc6e67a..d2cdd40aa94 100644 --- a/tests/e2e/cost_calculation/cases.json +++ b/tests/e2e/cost_calculation/cases.json @@ -1,468 +1,1837 @@ { "deployments": [ - {"map_key": "azure/gpt-5.4-mini", "litellm_model": "azure/cc-pinned-deployment", "base_model": "azure/gpt-5.4-mini"} + { + "map_key": "azure/gpt-5.4-mini", + "litellm_model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + } ], "cases": [ { - "name": "basic", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "name": "input_text", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "owns": [ + "input_cost_per_token", + "output_cost_per_token" + ], + "fallback_for": [], "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.034, "input_cost": 0.0204, "output_cost": 0.0136, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.03, "input_cost": 0.018, "output_cost": 0.012, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-haiku-4-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-opus-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-sonnet-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/kimi-k3": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/qwen3p8-max": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, - "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0228, "output_cost": 0.0152, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.036, "input_cost": 0.0216, "output_cost": 0.0144, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.6": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { "name": "cache_read", - "usage": {"fresh_input_tokens": 100, "cache_read_tokens": 50, "output_tokens": 30}, + "family": "pricing", + "usage": { + "fresh_input_tokens": 640, + "cache_read_tokens": 12288, + "output_tokens": 380 + }, + "owns": [ + "cache_read_input_token_cost" + ], + "fallback_for": [], "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.02805, "input_cost": 0.01785, "output_cost": 0.0102, "prompt_tokens": 150, "completion_tokens": 30}, - "azure/gpt-5.4-mini": {"spend": 0.0264, "input_cost": 0.0168, "output_cost": 0.0096, "prompt_tokens": 150, "completion_tokens": 30}, - "azure/gpt-5.6": {"spend": 0.02475, "input_cost": 0.01575, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-haiku-4-5": {"spend": 0.01155, "input_cost": 0.00735, "output_cost": 0.0042, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-opus-5": {"spend": 0.00825, "input_cost": 0.00525, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-sonnet-5": {"spend": 0.0099, "input_cost": 0.0063, "output_cost": 0.0036, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0231, "input_cost": 0.0147, "output_cost": 0.0084, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/kimi-k3": {"spend": 0.0198, "input_cost": 0.0126, "output_cost": 0.0072, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/qwen3p8-max": {"spend": 0.02145, "input_cost": 0.01365, "output_cost": 0.0078, "prompt_tokens": 150, "completion_tokens": 30}, - "gemini-3.1-pro-preview": {"spend": 0.03465, "input_cost": 0.02205, "output_cost": 0.0126, "prompt_tokens": 150, "completion_tokens": 30}, - "gemini-3.8-flash": {"spend": 0.033, "input_cost": 0.021, "output_cost": 0.012, "prompt_tokens": 150, "completion_tokens": 30}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.01485, "input_cost": 0.00945, "output_cost": 0.0054, "prompt_tokens": 150, "completion_tokens": 30}, - "gemini/gemini-3.8-flash": {"spend": 0.0132, "input_cost": 0.0084, "output_cost": 0.0048, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.3-codex": {"spend": 0.00495, "input_cost": 0.00315, "output_cost": 0.0018, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.4-mini": {"spend": 0.0066, "input_cost": 0.0042, "output_cost": 0.0024, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.5-pro": {"spend": 0.0033, "input_cost": 0.0021, "output_cost": 0.0012, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.6": {"spend": 0.00165, "input_cost": 0.00105, "output_cost": 0.0006, "prompt_tokens": 150, "completion_tokens": 30}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.0165, "input_cost": 0.0105, "output_cost": 0.006, "prompt_tokens": 150, "completion_tokens": 30}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.01815, "input_cost": 0.01155, "output_cost": 0.0066, "prompt_tokens": 150, "completion_tokens": 30}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0297, "input_cost": 0.0189, "output_cost": 0.0108, "prompt_tokens": 150, "completion_tokens": 30} + "gpt-5.6": { + "spend": 0.0085904, + "input_cost": 0.0032704, + "output_cost": 0.00532, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gpt-5.4-mini": { + "spend": 0.00171808, + "input_cost": 0.00065408, + "output_cost": 0.001064, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "azure/gpt-5.6": { + "spend": 0.00883584, + "input_cost": 0.00336384, + "output_cost": 0.005472, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "azure/gpt-5.4-mini": { + "spend": 0.001767168, + "input_cost": 0.000672768, + "output_cost": 0.0010944, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gpt-5.3-codex": { + "spend": 0.0073632, + "input_cost": 0.0028032, + "output_cost": 0.00456, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gpt-5.5-pro": { + "spend": 0.073632, + "input_cost": 0.028032, + "output_cost": 0.0456, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "claude-opus-5": { + "spend": 0.018844, + "input_cost": 0.009344, + "output_cost": 0.0095, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "claude-sonnet-5": { + "spend": 0.0113064, + "input_cost": 0.0056064, + "output_cost": 0.0057, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "claude-haiku-4-5": { + "spend": 0.0037688, + "input_cost": 0.0018688, + "output_cost": 0.0019, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.0207284, + "input_cost": 0.0102784, + "output_cost": 0.01045, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01243704, + "input_cost": 0.00616704, + "output_cost": 0.00627, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.0082976, + "input_cost": 0.0037376, + "output_cost": 0.00456, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.0020744, + "input_cost": 0.0009344, + "output_cost": 0.00114, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gemini-3.1-pro": { + "spend": 0.00871248, + "input_cost": 0.00392448, + "output_cost": 0.004788, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "gemini-3.8-flash": { + "spend": 0.002157376, + "input_cost": 0.000971776, + "output_cost": 0.0011856, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.00207128, + "input_cost": 0.00112128, + "output_cost": 0.00095, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.00304992, + "input_cost": 0.00168192, + "output_cost": 0.001368, + "prompt_tokens": 12928, + "completion_tokens": 380 + } } }, { "name": "cache_write_5m", - "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 60, "output_tokens": 30}, + "family": "pricing", + "usage": { + "fresh_input_tokens": 512, + "cache_write_5m_tokens": 9216, + "output_tokens": 350 + }, + "owns": [ + "cache_creation_input_token_cost" + ], + "fallback_for": [], "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.0561, "input_cost": 0.0459, "output_cost": 0.0102, "prompt_tokens": 150, "completion_tokens": 30}, - "azure/gpt-5.4-mini": {"spend": 0.0528, "input_cost": 0.0432, "output_cost": 0.0096, "prompt_tokens": 150, "completion_tokens": 30}, - "azure/gpt-5.6": {"spend": 0.0495, "input_cost": 0.0405, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-haiku-4-5": {"spend": 0.0231, "input_cost": 0.0189, "output_cost": 0.0042, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-opus-5": {"spend": 0.0165, "input_cost": 0.0135, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-sonnet-5": {"spend": 0.0198, "input_cost": 0.0162, "output_cost": 0.0036, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0408, "input_cost": 0.0324, "output_cost": 0.0084, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/kimi-k3": {"spend": 0.0378, "input_cost": 0.0306, "output_cost": 0.0072, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/qwen3p8-max": {"spend": 0.0393, "input_cost": 0.0315, "output_cost": 0.0078, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.4-mini": {"spend": 0.0132, "input_cost": 0.0108, "output_cost": 0.0024, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.6": {"spend": 0.0033, "input_cost": 0.0027, "output_cost": 0.0006, "prompt_tokens": 150, "completion_tokens": 30}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.033, "input_cost": 0.027, "output_cost": 0.006, "prompt_tokens": 150, "completion_tokens": 30}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0363, "input_cost": 0.0297, "output_cost": 0.0066, "prompt_tokens": 150, "completion_tokens": 30}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0594, "input_cost": 0.0486, "output_cost": 0.0108, "prompt_tokens": 150, "completion_tokens": 30} + "claude-opus-5": { + "spend": 0.06891, + "input_cost": 0.06016, + "output_cost": 0.00875, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "claude-sonnet-5": { + "spend": 0.041346, + "input_cost": 0.036096, + "output_cost": 0.00525, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "claude-haiku-4-5": { + "spend": 0.013782, + "input_cost": 0.012032, + "output_cost": 0.00175, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.075801, + "input_cost": 0.066176, + "output_cost": 0.009625, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.0454806, + "input_cost": 0.0397056, + "output_cost": 0.005775, + "prompt_tokens": 9728, + "completion_tokens": 350 + } } }, { "name": "cache_write_1h", - "usage": {"fresh_input_tokens": 90, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 40, "output_tokens": 30}, + "family": "pricing", + "usage": { + "fresh_input_tokens": 512, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 7168, + "output_tokens": 350 + }, + "owns": [ + "cache_creation_input_token_cost_above_1hr" + ], + "fallback_for": [], "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.0629, "input_cost": 0.0527, "output_cost": 0.0102, "prompt_tokens": 150, "completion_tokens": 30}, - "azure/gpt-5.4-mini": {"spend": 0.0592, "input_cost": 0.0496, "output_cost": 0.0096, "prompt_tokens": 150, "completion_tokens": 30}, - "azure/gpt-5.6": {"spend": 0.0555, "input_cost": 0.0465, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-haiku-4-5": {"spend": 0.0259, "input_cost": 0.0217, "output_cost": 0.0042, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-opus-5": {"spend": 0.0185, "input_cost": 0.0155, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 30}, - "claude-sonnet-5": {"spend": 0.0222, "input_cost": 0.0186, "output_cost": 0.0036, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0452, "input_cost": 0.0368, "output_cost": 0.0084, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/kimi-k3": {"spend": 0.0422, "input_cost": 0.035, "output_cost": 0.0072, "prompt_tokens": 150, "completion_tokens": 30}, - "fireworks_ai/qwen3p8-max": {"spend": 0.0437, "input_cost": 0.0359, "output_cost": 0.0078, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.4-mini": {"spend": 0.0148, "input_cost": 0.0124, "output_cost": 0.0024, "prompt_tokens": 150, "completion_tokens": 30}, - "gpt-5.6": {"spend": 0.0037, "input_cost": 0.0031, "output_cost": 0.0006, "prompt_tokens": 150, "completion_tokens": 30}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.037, "input_cost": 0.031, "output_cost": 0.006, "prompt_tokens": 150, "completion_tokens": 30}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0407, "input_cost": 0.0341, "output_cost": 0.0066, "prompt_tokens": 150, "completion_tokens": 30}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0666, "input_cost": 0.0558, "output_cost": 0.0108, "prompt_tokens": 150, "completion_tokens": 30} + "claude-opus-5": { + "spend": 0.09579, + "input_cost": 0.08704, + "output_cost": 0.00875, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "claude-sonnet-5": { + "spend": 0.057474, + "input_cost": 0.052224, + "output_cost": 0.00525, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "claude-haiku-4-5": { + "spend": 0.019158, + "input_cost": 0.017408, + "output_cost": 0.00175, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.105369, + "input_cost": 0.095744, + "output_cost": 0.009625, + "prompt_tokens": 9728, + "completion_tokens": 350 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.0632214, + "input_cost": 0.0574464, + "output_cost": 0.005775, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + } + }, + { + "name": "audio_input", + "family": "pricing", + "usage": { + "fresh_input_tokens": 96, + "audio_input_tokens": 1450, + "output_tokens": 210 + }, + "owns": [ + "input_cost_per_audio_token" + ], + "fallback_for": [], + "audio_input": true, + "expected": { + "gpt-5.6": { + "spend": 0.061108, + "input_cost": 0.058168, + "output_cost": 0.00294, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "gpt-5.4-mini": { + "spend": 0.0151216, + "input_cost": 0.0145336, + "output_cost": 0.000588, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "azure/gpt-5.6": { + "spend": 0.0626468, + "input_cost": 0.0596228, + "output_cost": 0.003024, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "azure/gpt-5.4-mini": { + "spend": 0.01586436, + "input_cost": 0.01525956, + "output_cost": 0.0006048, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.006482, + "input_cost": 0.003962, + "output_cost": 0.00252, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.002128, + "input_cost": 0.001498, + "output_cost": 0.00063, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "gemini-3.1-pro": { + "spend": 0.0067626, + "input_cost": 0.0041166, + "output_cost": 0.002646, + "prompt_tokens": 1546, + "completion_tokens": 210 + }, + "gemini-3.8-flash": { + "spend": 0.00221312, + "input_cost": 0.00155792, + "output_cost": 0.0006552, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + } + }, + { + "name": "audio_output", + "family": "pricing", + "usage": { + "fresh_input_tokens": 220, + "output_tokens": 180, + "audio_output_tokens": 1120 + }, + "owns": [ + "output_cost_per_audio_token" + ], + "fallback_for": [], + "audio_output": true, + "expected": { + "gpt-5.6": { + "spend": 0.092505, + "input_cost": 0.000385, + "output_cost": 0.09212, + "prompt_tokens": 220, + "completion_tokens": 1300 + }, + "gpt-5.4-mini": { + "spend": 0.022981, + "input_cost": 7.7e-05, + "output_cost": 0.022904, + "prompt_tokens": 220, + "completion_tokens": 1300 + }, + "azure/gpt-5.6": { + "spend": 0.094828, + "input_cost": 0.000396, + "output_cost": 0.094432, + "prompt_tokens": 220, + "completion_tokens": 1300 + }, + "azure/gpt-5.4-mini": { + "spend": 0.0241176, + "input_cost": 7.92e-05, + "output_cost": 0.0240384, + "prompt_tokens": 220, + "completion_tokens": 1300 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.00737, + "input_cost": 0.00011, + "output_cost": 0.00726, + "prompt_tokens": 220, + "completion_tokens": 1300 + }, + "gemini-3.8-flash": { + "spend": 0.0076648, + "input_cost": 0.0001144, + "output_cost": 0.0075504, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + } + }, + { + "name": "image_input", + "family": "pricing", + "usage": { + "fresh_input_tokens": 310, + "image_input_tokens": 1806, + "output_tokens": 240 + }, + "owns": [ + "input_cost_per_image_token" + ], + "fallback_for": [], + "image_input": true, + "expected": { + "gemini/gemini-3.1-pro": { + "spend": 0.0074732, + "input_cost": 0.0045932, + "output_cost": 0.00288, + "prompt_tokens": 2116, + "completion_tokens": 240 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.0018683, + "input_cost": 0.0011483, + "output_cost": 0.00072, + "prompt_tokens": 2116, + "completion_tokens": 240 + }, + "gemini-3.1-pro": { + "spend": 0.0078288, + "input_cost": 0.0048048, + "output_cost": 0.003024, + "prompt_tokens": 2116, + "completion_tokens": 240 + } + } + }, + { + "name": "video_input", + "family": "pricing", + "usage": { + "fresh_input_tokens": 140, + "video_input_tokens": 7920, + "output_tokens": 300 + }, + "owns": [ + "input_cost_per_video_token" + ], + "fallback_for": [], + "video_input": true, + "expected": { + "gemini/gemini-3.1-pro": { + "spend": 0.022888, + "input_cost": 0.019288, + "output_cost": 0.0036, + "prompt_tokens": 8060, + "completion_tokens": 300 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.005722, + "input_cost": 0.004822, + "output_cost": 0.0009, + "prompt_tokens": 8060, + "completion_tokens": 300 + }, + "gemini-3.8-flash": { + "spend": 0.0059192, + "input_cost": 0.0049832, + "output_cost": 0.000936, + "prompt_tokens": 8060, + "completion_tokens": 300 + } } }, { "name": "reasoning", - "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "reasoning_tokens": 70}, + "family": "pricing", + "usage": { + "fresh_input_tokens": 1240, + "output_tokens": 560, + "reasoning_tokens": 3480 + }, + "owns": [ + "output_cost_per_reasoning_token" + ], + "fallback_for": [], + "reasoning": true, "expected": { - "azure/gpt-5.4-mini": {"spend": 0.0816, "input_cost": 0.016, "output_cost": 0.0656, "prompt_tokens": 100, "completion_tokens": 100}, - "azure/gpt-5.6": {"spend": 0.0765, "input_cost": 0.015, "output_cost": 0.0615, "prompt_tokens": 100, "completion_tokens": 100}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0609, "input_cost": 0.014, "output_cost": 0.0469, "prompt_tokens": 100, "completion_tokens": 100}, - "fireworks_ai/kimi-k3": {"spend": 0.0577, "input_cost": 0.012, "output_cost": 0.0457, "prompt_tokens": 100, "completion_tokens": 100}, - "fireworks_ai/qwen3p8-max": {"spend": 0.0593, "input_cost": 0.013, "output_cost": 0.0463, "prompt_tokens": 100, "completion_tokens": 100}, - "gemini-3.1-pro-preview": {"spend": 0.1071, "input_cost": 0.021, "output_cost": 0.0861, "prompt_tokens": 100, "completion_tokens": 100}, - "gemini-3.8-flash": {"spend": 0.102, "input_cost": 0.02, "output_cost": 0.082, "prompt_tokens": 100, "completion_tokens": 100}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.0459, "input_cost": 0.009, "output_cost": 0.0369, "prompt_tokens": 100, "completion_tokens": 100}, - "gemini/gemini-3.8-flash": {"spend": 0.0408, "input_cost": 0.008, "output_cost": 0.0328, "prompt_tokens": 100, "completion_tokens": 100}, - "gpt-5.3-codex": {"spend": 0.0153, "input_cost": 0.003, "output_cost": 0.0123, "prompt_tokens": 100, "completion_tokens": 100}, - "gpt-5.4-mini": {"spend": 0.0204, "input_cost": 0.004, "output_cost": 0.0164, "prompt_tokens": 100, "completion_tokens": 100}, - "gpt-5.5-pro": {"spend": 0.0102, "input_cost": 0.002, "output_cost": 0.0082, "prompt_tokens": 100, "completion_tokens": 100}, - "gpt-5.6": {"spend": 0.0051, "input_cost": 0.001, "output_cost": 0.0041, "prompt_tokens": 100, "completion_tokens": 100}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.051, "input_cost": 0.01, "output_cost": 0.041, "prompt_tokens": 100, "completion_tokens": 100}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0561, "input_cost": 0.011, "output_cost": 0.0451, "prompt_tokens": 100, "completion_tokens": 100} + "gpt-5.6": { + "spend": 0.06569, + "input_cost": 0.00217, + "output_cost": 0.06352, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "gpt-5.4-mini": { + "spend": 0.013138, + "input_cost": 0.000434, + "output_cost": 0.012704, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "azure/gpt-5.6": { + "spend": 0.067716, + "input_cost": 0.002232, + "output_cost": 0.065484, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "azure/gpt-5.4-mini": { + "spend": 0.0135432, + "input_cost": 0.0004464, + "output_cost": 0.0130968, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "gpt-5.3-codex": { + "spend": 0.05382, + "input_cost": 0.00186, + "output_cost": 0.05196, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "gpt-5.5-pro": { + "spend": 0.5382, + "input_cost": 0.0186, + "output_cost": 0.5196, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.05444, + "input_cost": 0.00248, + "output_cost": 0.05196, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.01448, + "input_cost": 0.00062, + "output_cost": 0.01386, + "prompt_tokens": 1240, + "completion_tokens": 4040 + }, + "gemini-3.1-pro": { + "spend": 0.05664, + "input_cost": 0.002604, + "output_cost": 0.054036, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } } }, { - "name": "audio", - "usage": {"fresh_input_tokens": 100, "audio_input_tokens": 25, "output_tokens": 30, "audio_output_tokens": 15}, + "name": "tiered_input_above_200k", + "family": "pricing", + "usage": { + "fresh_input_tokens": 204800, + "output_tokens": 620 + }, + "owns": [ + "input_cost_per_token_above_200k_tokens", + "output_cost_per_token_above_200k_tokens" + ], + "fallback_for": [], "expected": { - "azure/gpt-5.4-mini": {"spend": 0.0664, "input_cost": 0.04, "output_cost": 0.0264, "prompt_tokens": 125, "completion_tokens": 45}, - "azure/gpt-5.6": {"spend": 0.06225, "input_cost": 0.0375, "output_cost": 0.02475, "prompt_tokens": 125, "completion_tokens": 45}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.05045, "input_cost": 0.0305, "output_cost": 0.01995, "prompt_tokens": 125, "completion_tokens": 45}, - "fireworks_ai/kimi-k3": {"spend": 0.04725, "input_cost": 0.0285, "output_cost": 0.01875, "prompt_tokens": 125, "completion_tokens": 45}, - "fireworks_ai/qwen3p8-max": {"spend": 0.04885, "input_cost": 0.0295, "output_cost": 0.01935, "prompt_tokens": 125, "completion_tokens": 45}, - "gemini-3.1-pro-preview": {"spend": 0.08715, "input_cost": 0.0525, "output_cost": 0.03465, "prompt_tokens": 125, "completion_tokens": 45}, - "gemini-3.8-flash": {"spend": 0.083, "input_cost": 0.05, "output_cost": 0.033, "prompt_tokens": 125, "completion_tokens": 45}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.03735, "input_cost": 0.0225, "output_cost": 0.01485, "prompt_tokens": 125, "completion_tokens": 45}, - "gemini/gemini-3.8-flash": {"spend": 0.0332, "input_cost": 0.02, "output_cost": 0.0132, "prompt_tokens": 125, "completion_tokens": 45}, - "gpt-5.4-mini": {"spend": 0.0166, "input_cost": 0.01, "output_cost": 0.0066, "prompt_tokens": 125, "completion_tokens": 45}, - "gpt-5.6": {"spend": 0.00415, "input_cost": 0.0025, "output_cost": 0.00165, "prompt_tokens": 125, "completion_tokens": 45}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.0415, "input_cost": 0.025, "output_cost": 0.0165, "prompt_tokens": 125, "completion_tokens": 45}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.04565, "input_cost": 0.0275, "output_cost": 0.01815, "prompt_tokens": 125, "completion_tokens": 45} + "claude-opus-5": { + "spend": 2.07125, + "input_cost": 2.048, + "output_cost": 0.02325, + "prompt_tokens": 204800, + "completion_tokens": 620 + }, + "claude-sonnet-5": { + "spend": 1.24275, + "input_cost": 1.2288, + "output_cost": 0.01395, + "prompt_tokens": 204800, + "completion_tokens": 620 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 2.278375, + "input_cost": 2.2528, + "output_cost": 0.025575, + "prompt_tokens": 204800, + "completion_tokens": 620 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.83036, + "input_cost": 0.8192, + "output_cost": 0.01116, + "prompt_tokens": 204800, + "completion_tokens": 620 + }, + "gemini-3.1-pro": { + "spend": 0.871878, + "input_cost": 0.86016, + "output_cost": 0.011718, + "prompt_tokens": 204800, + "completion_tokens": 620 + } } }, { - "name": "tiered", - "usage": {"fresh_input_tokens": 200001, "output_tokens": 30}, + "name": "tiered_cache_read_above_200k", + "family": "pricing", + "usage": { + "fresh_input_tokens": 4096, + "cache_read_tokens": 201728, + "output_tokens": 480 + }, + "owns": [ + "cache_read_input_token_cost_above_200k_tokens" + ], + "fallback_for": [], "expected": { - "azure/gpt-5.4-mini": {"spend": 256.04448, "input_cost": 256.00128, "output_cost": 0.0432, "prompt_tokens": 200001, "completion_tokens": 30}, - "azure/gpt-5.6": {"spend": 240.0417, "input_cost": 240.0012, "output_cost": 0.0405, "prompt_tokens": 200001, "completion_tokens": 30}, - "gemini-3.1-pro-preview": {"spend": 336.05838, "input_cost": 336.00168, "output_cost": 0.0567, "prompt_tokens": 200001, "completion_tokens": 30}, - "gemini-3.8-flash": {"spend": 320.0556, "input_cost": 320.0016, "output_cost": 0.054, "prompt_tokens": 200001, "completion_tokens": 30}, - "gemini/gemini-3.1-pro-preview": {"spend": 144.02502, "input_cost": 144.00072, "output_cost": 0.0243, "prompt_tokens": 200001, "completion_tokens": 30}, - "gemini/gemini-3.8-flash": {"spend": 128.02224, "input_cost": 128.00064, "output_cost": 0.0216, "prompt_tokens": 200001, "completion_tokens": 30}, - "gpt-5.3-codex": {"spend": 48.00834, "input_cost": 48.00024, "output_cost": 0.0081, "prompt_tokens": 200001, "completion_tokens": 30}, - "gpt-5.4-mini": {"spend": 64.01112, "input_cost": 64.00032, "output_cost": 0.0108, "prompt_tokens": 200001, "completion_tokens": 30}, - "gpt-5.5-pro": {"spend": 32.00556, "input_cost": 32.00016, "output_cost": 0.0054, "prompt_tokens": 200001, "completion_tokens": 30}, - "gpt-5.6": {"spend": 16.00278, "input_cost": 16.00008, "output_cost": 0.0027, "prompt_tokens": 200001, "completion_tokens": 30}, - "together_ai/moonshotai/Kimi-K3": {"spend": 160.0278, "input_cost": 160.0008, "output_cost": 0.027, "prompt_tokens": 200001, "completion_tokens": 30}, - "together_ai/zai-org/GLM-5.3": {"spend": 176.03058, "input_cost": 176.00088, "output_cost": 0.0297, "prompt_tokens": 200001, "completion_tokens": 30} + "claude-opus-5": { + "spend": 0.260688, + "input_cost": 0.242688, + "output_cost": 0.018, + "prompt_tokens": 205824, + "completion_tokens": 480 + }, + "claude-sonnet-5": { + "spend": 0.1564128, + "input_cost": 0.1456128, + "output_cost": 0.0108, + "prompt_tokens": 205824, + "completion_tokens": 480 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.2867568, + "input_cost": 0.2669568, + "output_cost": 0.0198, + "prompt_tokens": 205824, + "completion_tokens": 480 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.1057152, + "input_cost": 0.0970752, + "output_cost": 0.00864, + "prompt_tokens": 205824, + "completion_tokens": 480 + }, + "gemini-3.1-pro": { + "spend": 0.11100096, + "input_cost": 0.10192896, + "output_cost": 0.009072, + "prompt_tokens": 205824, + "completion_tokens": 480 + } + } + }, + { + "name": "tiered_cache_write_above_200k", + "family": "pricing", + "usage": { + "fresh_input_tokens": 4096, + "cache_write_5m_tokens": 200704, + "output_tokens": 480 + }, + "owns": [ + "cache_creation_input_token_cost_above_200k_tokens" + ], + "fallback_for": [], + "expected": { + "claude-opus-5": { + "spend": 2.56776, + "input_cost": 2.54976, + "output_cost": 0.018, + "prompt_tokens": 204800, + "completion_tokens": 480 + }, + "claude-sonnet-5": { + "spend": 1.540656, + "input_cost": 1.529856, + "output_cost": 0.0108, + "prompt_tokens": 204800, + "completion_tokens": 480 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 2.824536, + "input_cost": 2.804736, + "output_cost": 0.0198, + "prompt_tokens": 204800, + "completion_tokens": 480 + } } }, { "name": "service_tier_flex", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "owns": [ + "input_cost_per_token_flex", + "output_cost_per_token_flex" + ], + "fallback_for": [], "service_tier": "flex", "expected": { - "azure/gpt-5.4-mini": {"spend": 0.0448, "input_cost": 0.0288, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.042, "input_cost": 0.027, "output_cost": 0.015, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.0588, "input_cost": 0.0378, "output_cost": 0.021, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.056, "input_cost": 0.036, "output_cost": 0.02, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.0252, "input_cost": 0.0162, "output_cost": 0.009, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.0224, "input_cost": 0.0144, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.0084, "input_cost": 0.0054, "output_cost": 0.003, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.0112, "input_cost": 0.0072, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.0056, "input_cost": 0.0036, "output_cost": 0.002, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.0028, "input_cost": 0.0018, "output_cost": 0.001, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.028, "input_cost": 0.018, "output_cost": 0.01, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0308, "input_cost": 0.0198, "output_cost": 0.011, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.6": { + "spend": 0.004494, + "input_cost": 0.00161, + "output_cost": 0.002884, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0008988, + "input_cost": 0.000322, + "output_cost": 0.0005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0046224, + "input_cost": 0.001656, + "output_cost": 0.0029664, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00092448, + "input_cost": 0.0003312, + "output_cost": 0.00059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.003852, + "input_cost": 0.00138, + "output_cost": 0.002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.03852, + "input_cost": 0.0138, + "output_cost": 0.02472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.010725, + "input_cost": 0.00506, + "output_cost": 0.005665, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.006435, + "input_cost": 0.003036, + "output_cost": 0.003399, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.004312, + "input_cost": 0.00184, + "output_cost": 0.002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.001078, + "input_cost": 0.00046, + "output_cost": 0.000618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.0045276, + "input_cost": 0.001932, + "output_cost": 0.0025956, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.00112112, + "input_cost": 0.0004784, + "output_cost": 0.00064272, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { "name": "service_tier_priority", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "owns": [ + "input_cost_per_token_priority", + "output_cost_per_token_priority" + ], + "fallback_for": [], "service_tier": "priority", "expected": { - "azure/gpt-5.4-mini": {"spend": 0.04992, "input_cost": 0.03264, "output_cost": 0.01728, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.0468, "input_cost": 0.0306, "output_cost": 0.0162, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.06552, "input_cost": 0.04284, "output_cost": 0.02268, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.0624, "input_cost": 0.0408, "output_cost": 0.0216, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.02808, "input_cost": 0.01836, "output_cost": 0.00972, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.02496, "input_cost": 0.01632, "output_cost": 0.00864, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.00936, "input_cost": 0.00612, "output_cost": 0.00324, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.01248, "input_cost": 0.00816, "output_cost": 0.00432, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.00624, "input_cost": 0.00408, "output_cost": 0.00216, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.00312, "input_cost": 0.00204, "output_cost": 0.00108, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.0312, "input_cost": 0.0204, "output_cost": 0.0108, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.03432, "input_cost": 0.02244, "output_cost": 0.01188, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.6": { + "spend": 0.017976, + "input_cost": 0.00644, + "output_cost": 0.011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0035952, + "input_cost": 0.001288, + "output_cost": 0.0023072, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0184896, + "input_cost": 0.006624, + "output_cost": 0.0118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00369792, + "input_cost": 0.0013248, + "output_cost": 0.00237312, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.015408, + "input_cost": 0.00552, + "output_cost": 0.009888, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.15408, + "input_cost": 0.0552, + "output_cost": 0.09888, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.024375, + "input_cost": 0.0115, + "output_cost": 0.012875, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.014625, + "input_cost": 0.0069, + "output_cost": 0.007725, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.004875, + "input_cost": 0.0023, + "output_cost": 0.002575, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.0268125, + "input_cost": 0.01265, + "output_cost": 0.0141625, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.0160875, + "input_cost": 0.00759, + "output_cost": 0.0084975, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.01078, + "input_cost": 0.0046, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.002695, + "input_cost": 0.00115, + "output_cost": 0.001545, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.011319, + "input_cost": 0.00483, + "output_cost": 0.006489, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.0028028, + "input_cost": 0.001196, + "output_cost": 0.0016068, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { - "name": "web_search", - "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 3}, + "name": "anthropic_fast_mode", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "owns": [ + "provider_specific_entry.fast" + ], + "fallback_for": [], + "speed": "fast", "expected": { - "claude-haiku-4-5": {"spend": 0.0712, "input_cost": 0.007, "output_cost": 0.0042, "prompt_tokens": 100, "completion_tokens": 30}, - "claude-opus-5": {"spend": 0.068, "input_cost": 0.005, "output_cost": 0.003, "prompt_tokens": 100, "completion_tokens": 30}, - "claude-sonnet-5": {"spend": 0.0696, "input_cost": 0.006, "output_cost": 0.0036, "prompt_tokens": 100, "completion_tokens": 30}, - "gemini-3.1-pro-preview": {"spend": 0.0936, "input_cost": 0.021, "output_cost": 0.0126, "prompt_tokens": 100, "completion_tokens": 30}, - "gemini-3.8-flash": {"spend": 0.092, "input_cost": 0.02, "output_cost": 0.012, "prompt_tokens": 100, "completion_tokens": 30}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.0744, "input_cost": 0.009, "output_cost": 0.0054, "prompt_tokens": 100, "completion_tokens": 30}, - "gemini/gemini-3.8-flash": {"spend": 0.0728, "input_cost": 0.008, "output_cost": 0.0048, "prompt_tokens": 100, "completion_tokens": 30}, - "gpt-5.3-codex": {"spend": 0.0648, "input_cost": 0.003, "output_cost": 0.0018, "prompt_tokens": 100, "completion_tokens": 30}, - "gpt-5.5-pro": {"spend": 0.0632, "input_cost": 0.002, "output_cost": 0.0012, "prompt_tokens": 100, "completion_tokens": 30} + "claude-opus-5": { + "spend": 0.117, + "input_cost": 0.0552, + "output_cost": 0.0618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { - "name": "web_search_single", - "usage": {"fresh_input_tokens": 100, "output_tokens": 30, "web_search_calls": 1}, + "name": "anthropic_us_inference", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "owns": [ + "provider_specific_entry.us" + ], + "fallback_for": [], + "inference_geo": "us", "expected": { - "azure/gpt-5.4-mini": {"spend": 0.0456, "input_cost": 0.016, "output_cost": 0.0096, "prompt_tokens": 100, "completion_tokens": 30}, - "azure/gpt-5.6": {"spend": 0.044, "input_cost": 0.015, "output_cost": 0.009, "prompt_tokens": 100, "completion_tokens": 30}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0424, "input_cost": 0.014, "output_cost": 0.0084, "prompt_tokens": 100, "completion_tokens": 30}, - "fireworks_ai/kimi-k3": {"spend": 0.0392, "input_cost": 0.012, "output_cost": 0.0072, "prompt_tokens": 100, "completion_tokens": 30}, - "fireworks_ai/qwen3p8-max": {"spend": 0.0408, "input_cost": 0.013, "output_cost": 0.0078, "prompt_tokens": 100, "completion_tokens": 30}, - "gpt-5.4-mini": {"spend": 0.0264, "input_cost": 0.004, "output_cost": 0.0024, "prompt_tokens": 100, "completion_tokens": 30}, - "gpt-5.6": {"spend": 0.0216, "input_cost": 0.001, "output_cost": 0.0006, "prompt_tokens": 100, "completion_tokens": 30}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.036, "input_cost": 0.01, "output_cost": 0.006, "prompt_tokens": 100, "completion_tokens": 30}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0376, "input_cost": 0.011, "output_cost": 0.0066, "prompt_tokens": 100, "completion_tokens": 30} + "claude-opus-5": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.00429, + "input_cost": 0.002024, + "output_cost": 0.002266, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "web_search_medium", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "web_search_calls": 3 + }, + "owns": [ + "search_context_cost_per_query.search_context_size_medium", + "web_search_billing_unit" + ], + "fallback_for": [], + "web_search": "medium", + "expected": { + "gpt-5.6": { + "spend": 0.021488, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0142976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0217448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.01434896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.045204, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.11454, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0495, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0417, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0339, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.113624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.1140552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "web_search_low", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "web_search_calls": 1 + }, + "owns": [ + "search_context_cost_per_query.search_context_size_low" + ], + "fallback_for": [], + "web_search": "low", + "expected": { + "gpt-5.6": { + "spend": 0.018988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0117976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0192448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.01184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.017704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.08704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "web_search_high", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "web_search_calls": 1 + }, + "owns": [ + "search_context_cost_per_query.search_context_size_high" + ], + "fallback_for": [], + "web_search": "high", + "expected": { + "gpt-5.6": { + "spend": 0.023988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0167976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0242448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.01684896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.022704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.09204, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "web_search_per_prompt", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "web_search_calls": 3 + }, + "owns": [ + "search_context_cost_per_query.search_context_size_medium", + "web_search_billing_unit" + ], + "fallback_for": [], + "web_search": "medium", + "expected": { + "gemini/gemini-3.8-flash": { + "spend": 0.037156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.03724224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "google_maps_grounding", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "google_maps_calls": 1 + }, + "owns": [ + "google_maps_grounding_cost_per_query" + ], + "fallback_for": [], + "google_maps": true, + "expected": { + "gemini/gemini-3.1-pro": { + "spend": 0.033624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.027156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.0340552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.02724224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "file_search", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "file_search_calls": 1 + }, + "owns": [ + "file_search_cost_per_1k_calls" + ], + "fallback_for": [], + "file_search": true, + "expected": { + "gpt-5.3-codex": { + "spend": 0.010204, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07954, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "fallback_cache_read_at_input_rate", + "family": "pricing", + "usage": { + "fresh_input_tokens": 640, + "cache_read_tokens": 12288, + "output_tokens": 380 + }, + "owns": [], + "fallback_for": [ + "cache_read_input_token_cost" + ], + "expected": { + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00347132, + "input_cost": 0.00310272, + "output_cost": 0.0003686, + "prompt_tokens": 12928, + "completion_tokens": 380 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.0021672, + "input_cost": 0.0019392, + "output_cost": 0.000228, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + } + }, + { + "name": "fallback_cache_write_at_input_rate", + "family": "pricing", + "usage": { + "fresh_input_tokens": 512, + "cache_write_5m_tokens": 9216, + "output_tokens": 350 + }, + "owns": [], + "fallback_for": [ + "cache_creation_input_token_cost" + ], + "expected": { + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00267422, + "input_cost": 0.00233472, + "output_cost": 0.0003395, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + } + }, + { + "name": "fallback_reasoning_at_output_rate", + "family": "pricing", + "usage": { + "fresh_input_tokens": 1240, + "output_tokens": 560, + "reasoning_tokens": 3480 + }, + "owns": [], + "fallback_for": [ + "output_cost_per_reasoning_token" + ], + "reasoning": true, + "expected": { + "gemini-3.8-flash": { + "spend": 0.0132496, + "input_cost": 0.0006448, + "output_cost": 0.0126048, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + } + }, + { + "name": "fallback_image_tokens_at_input_rate", + "family": "pricing", + "usage": { + "fresh_input_tokens": 310, + "image_input_tokens": 1806, + "output_tokens": 240 + }, + "owns": [], + "fallback_for": [ + "input_cost_per_image_token" + ], + "image_input": true, + "expected": { + "gemini-3.8-flash": { + "spend": 0.00184912, + "input_cost": 0.00110032, + "output_cost": 0.0007488, + "prompt_tokens": 2116, + "completion_tokens": 240 + } + } + }, + { + "name": "fallback_video_tokens_at_input_rate", + "family": "pricing", + "usage": { + "fresh_input_tokens": 140, + "video_input_tokens": 7920, + "output_tokens": 300 + }, + "owns": [], + "fallback_for": [ + "input_cost_per_video_token" + ], + "video_input": true, + "expected": { + "gemini-3.1-pro": { + "spend": 0.020706, + "input_cost": 0.016926, + "output_cost": 0.00378, + "prompt_tokens": 8060, + "completion_tokens": 300 + } } }, { "name": "stream", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.034, "input_cost": 0.0204, "output_cost": 0.0136, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.03, "input_cost": 0.018, "output_cost": 0.012, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-haiku-4-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-opus-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-sonnet-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/kimi-k3": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/qwen3p8-max": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, - "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0228, "output_cost": 0.0152, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.036, "input_cost": 0.0216, "output_cost": 0.0144, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.6": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { "name": "stream_no_usage", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "stream_usage": "absent", "exact_spend": false, "models": [ - "anthropic.claude-sonnet-5-v1:0", - "azure/gpt-5.4-mini", + "gpt-5.6", + "gpt-5.4-mini", "azure/gpt-5.6", - "claude-haiku-4-5", + "azure/gpt-5.4-mini", + "gpt-5.3-codex", + "gpt-5.5-pro", "claude-opus-5", "claude-sonnet-5", - "fireworks_ai/deepseek-v4p1-flash", - "fireworks_ai/kimi-k3", - "fireworks_ai/qwen3p8-max", - "gemini-3.1-pro-preview", - "gemini-3.8-flash", - "gemini/gemini-3.1-pro-preview", - "gemini/gemini-3.8-flash", - "gpt-5.3-codex", - "gpt-5.4-mini", - "gpt-5.5-pro", - "gpt-5.6", + "claude-haiku-4-5", + "us.anthropic.claude-opus-5-v1:0", + "anthropic.claude-sonnet-5-v1:0", "meta.llama4-maverick-17b-instruct-v1:0", + "gemini/gemini-3.1-pro", + "gemini/gemini-3.8-flash", + "gemini-3.1-pro", + "gemini-3.8-flash", "together_ai/moonshotai/Kimi-K3", "together_ai/zai-org/GLM-5.3", - "us.anthropic.claude-opus-5-v1:0" + "fireworks_ai/accounts/fireworks/models/kimi-k3", + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "fireworks_ai/accounts/fireworks/models/qwen3p8-max" ] }, - { - "name": "response_model_override", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, - "response_model_override": true, - "expected": { - "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-haiku-4-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-opus-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-sonnet-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/kimi-k3": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/qwen3p8-max": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40} - } - }, - { - "name": "stream_response_model_override", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, - "stream": true, - "response_model_override": true, - "expected": { - "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-haiku-4-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-opus-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-sonnet-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/kimi-k3": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/qwen3p8-max": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40} - } - }, - { - "name": "tool_call", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, - "tool_call": true, - "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.034, "input_cost": 0.0204, "output_cost": 0.0136, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.4-mini": {"spend": 0.032, "input_cost": 0.0192, "output_cost": 0.0128, "prompt_tokens": 120, "completion_tokens": 40}, - "azure/gpt-5.6": {"spend": 0.03, "input_cost": 0.018, "output_cost": 0.012, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-haiku-4-5": {"spend": 0.014, "input_cost": 0.0084, "output_cost": 0.0056, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-opus-5": {"spend": 0.01, "input_cost": 0.006, "output_cost": 0.004, "prompt_tokens": 120, "completion_tokens": 40}, - "claude-sonnet-5": {"spend": 0.012, "input_cost": 0.0072, "output_cost": 0.0048, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.028, "input_cost": 0.0168, "output_cost": 0.0112, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/kimi-k3": {"spend": 0.024, "input_cost": 0.0144, "output_cost": 0.0096, "prompt_tokens": 120, "completion_tokens": 40}, - "fireworks_ai/qwen3p8-max": {"spend": 0.026, "input_cost": 0.0156, "output_cost": 0.0104, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.1-pro-preview": {"spend": 0.042, "input_cost": 0.0252, "output_cost": 0.0168, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini-3.8-flash": {"spend": 0.04, "input_cost": 0.024, "output_cost": 0.016, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.018, "input_cost": 0.0108, "output_cost": 0.0072, "prompt_tokens": 120, "completion_tokens": 40}, - "gemini/gemini-3.8-flash": {"spend": 0.016, "input_cost": 0.0096, "output_cost": 0.0064, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.4-mini": {"spend": 0.008, "input_cost": 0.0048, "output_cost": 0.0032, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.6": {"spend": 0.002, "input_cost": 0.0012, "output_cost": 0.0008, "prompt_tokens": 120, "completion_tokens": 40}, - "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0228, "output_cost": 0.0152, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.02, "input_cost": 0.012, "output_cost": 0.008, "prompt_tokens": 120, "completion_tokens": 40}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.022, "input_cost": 0.0132, "output_cost": 0.0088, "prompt_tokens": 120, "completion_tokens": 40}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.036, "input_cost": 0.0216, "output_cost": 0.0144, "prompt_tokens": 120, "completion_tokens": 40} - } - }, - { - "name": "stream_tool_call", - "usage": {"fresh_input_tokens": 80, "output_tokens": 25}, - "stream": true, - "tool_call": true, - "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.0221, "input_cost": 0.0136, "output_cost": 0.0085, "prompt_tokens": 80, "completion_tokens": 25}, - "azure/gpt-5.4-mini": {"spend": 0.0208, "input_cost": 0.0128, "output_cost": 0.008, "prompt_tokens": 80, "completion_tokens": 25}, - "azure/gpt-5.6": {"spend": 0.0195, "input_cost": 0.012, "output_cost": 0.0075, "prompt_tokens": 80, "completion_tokens": 25}, - "claude-haiku-4-5": {"spend": 0.0091, "input_cost": 0.0056, "output_cost": 0.0035, "prompt_tokens": 80, "completion_tokens": 25}, - "claude-opus-5": {"spend": 0.0065, "input_cost": 0.004, "output_cost": 0.0025, "prompt_tokens": 80, "completion_tokens": 25}, - "claude-sonnet-5": {"spend": 0.0078, "input_cost": 0.0048, "output_cost": 0.003, "prompt_tokens": 80, "completion_tokens": 25}, - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.0182, "input_cost": 0.0112, "output_cost": 0.007, "prompt_tokens": 80, "completion_tokens": 25}, - "fireworks_ai/kimi-k3": {"spend": 0.0156, "input_cost": 0.0096, "output_cost": 0.006, "prompt_tokens": 80, "completion_tokens": 25}, - "fireworks_ai/qwen3p8-max": {"spend": 0.0169, "input_cost": 0.0104, "output_cost": 0.0065, "prompt_tokens": 80, "completion_tokens": 25}, - "gemini-3.1-pro-preview": {"spend": 0.0273, "input_cost": 0.0168, "output_cost": 0.0105, "prompt_tokens": 80, "completion_tokens": 25}, - "gemini-3.8-flash": {"spend": 0.026, "input_cost": 0.016, "output_cost": 0.01, "prompt_tokens": 80, "completion_tokens": 25}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.0117, "input_cost": 0.0072, "output_cost": 0.0045, "prompt_tokens": 80, "completion_tokens": 25}, - "gemini/gemini-3.8-flash": {"spend": 0.0104, "input_cost": 0.0064, "output_cost": 0.004, "prompt_tokens": 80, "completion_tokens": 25}, - "gpt-5.3-codex": {"spend": 0.0039, "input_cost": 0.0024, "output_cost": 0.0015, "prompt_tokens": 80, "completion_tokens": 25}, - "gpt-5.4-mini": {"spend": 0.0052, "input_cost": 0.0032, "output_cost": 0.002, "prompt_tokens": 80, "completion_tokens": 25}, - "gpt-5.5-pro": {"spend": 0.0026, "input_cost": 0.0016, "output_cost": 0.001, "prompt_tokens": 80, "completion_tokens": 25}, - "gpt-5.6": {"spend": 0.0013, "input_cost": 0.0008, "output_cost": 0.0005, "prompt_tokens": 80, "completion_tokens": 25}, - "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.0247, "input_cost": 0.0152, "output_cost": 0.0095, "prompt_tokens": 80, "completion_tokens": 25}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.013, "input_cost": 0.008, "output_cost": 0.005, "prompt_tokens": 80, "completion_tokens": 25}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0143, "input_cost": 0.0088, "output_cost": 0.0055, "prompt_tokens": 80, "completion_tokens": 25}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.0234, "input_cost": 0.0144, "output_cost": 0.009, "prompt_tokens": 80, "completion_tokens": 25} - } - }, { "name": "stream_no_usage_tool_call", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "stream_usage": "absent", "tool_call": true, "exact_spend": false, "models": [ - "anthropic.claude-sonnet-5-v1:0", - "azure/gpt-5.4-mini", + "gpt-5.6", + "gpt-5.4-mini", "azure/gpt-5.6", - "claude-haiku-4-5", + "azure/gpt-5.4-mini", + "gpt-5.3-codex", + "gpt-5.5-pro", "claude-opus-5", "claude-sonnet-5", - "fireworks_ai/deepseek-v4p1-flash", - "fireworks_ai/kimi-k3", - "fireworks_ai/qwen3p8-max", - "gemini-3.1-pro-preview", - "gemini-3.8-flash", - "gemini/gemini-3.1-pro-preview", - "gemini/gemini-3.8-flash", - "gpt-5.3-codex", - "gpt-5.4-mini", - "gpt-5.5-pro", - "gpt-5.6", + "claude-haiku-4-5", + "us.anthropic.claude-opus-5-v1:0", + "anthropic.claude-sonnet-5-v1:0", "meta.llama4-maverick-17b-instruct-v1:0", + "gemini/gemini-3.1-pro", + "gemini/gemini-3.8-flash", + "gemini-3.1-pro", + "gemini-3.8-flash", "together_ai/moonshotai/Kimi-K3", "together_ai/zai-org/GLM-5.3", - "us.anthropic.claude-opus-5-v1:0" + "fireworks_ai/accounts/fireworks/models/kimi-k3", + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "fireworks_ai/accounts/fireworks/models/qwen3p8-max" ] }, { "name": "stream_no_usage_image_input", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "stream_usage": "absent", "image_input": true, "exact_spend": false, "models": [ - "anthropic.claude-sonnet-5-v1:0", - "azure/gpt-5.4-mini", + "gpt-5.6", + "gpt-5.4-mini", "azure/gpt-5.6", - "claude-haiku-4-5", + "azure/gpt-5.4-mini", + "gpt-5.3-codex", + "gpt-5.5-pro", "claude-opus-5", "claude-sonnet-5", - "fireworks_ai/deepseek-v4p1-flash", - "fireworks_ai/kimi-k3", - "fireworks_ai/qwen3p8-max", - "gemini-3.1-pro-preview", - "gemini-3.8-flash", - "gemini/gemini-3.1-pro-preview", - "gemini/gemini-3.8-flash", - "gpt-5.3-codex", - "gpt-5.4-mini", - "gpt-5.5-pro", - "gpt-5.6", + "claude-haiku-4-5", + "us.anthropic.claude-opus-5-v1:0", + "anthropic.claude-sonnet-5-v1:0", "meta.llama4-maverick-17b-instruct-v1:0", + "gemini/gemini-3.1-pro", + "gemini/gemini-3.8-flash", + "gemini-3.1-pro", + "gemini-3.8-flash", "together_ai/moonshotai/Kimi-K3", "together_ai/zai-org/GLM-5.3", - "us.anthropic.claude-opus-5-v1:0" + "fireworks_ai/accounts/fireworks/models/kimi-k3", + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "fireworks_ai/accounts/fireworks/models/qwen3p8-max" ] }, { "name": "stream_incomplete", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "terminal": "incomplete", "expected": { - "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.3-codex": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { "name": "stream_no_usage_incomplete", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "stream_usage": "absent", "terminal": "incomplete", @@ -474,17 +1843,37 @@ }, { "name": "stream_unvalidated", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "terminal": "unvalidated", "expected": { - "gpt-5.3-codex": {"spend": 0.006, "input_cost": 0.0036, "output_cost": 0.0024, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.004, "input_cost": 0.0024, "output_cost": 0.0016, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.3-codex": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { "name": "stream_no_usage_unvalidated", - "usage": {"fresh_input_tokens": 120, "output_tokens": 40}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "stream": true, "stream_usage": "absent", "terminal": "unvalidated", @@ -496,88 +1885,1008 @@ }, { "name": "prompt_blocked", - "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840 + }, "terminal": "prompt_blocked", - "response_model_override": true, "expected": { - "gemini-3.1-pro-preview": {"spend": 0.2, "input_cost": 0.2, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, - "gemini-3.8-flash": {"spend": 0.21, "input_cost": 0.21, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.08, "input_cost": 0.08, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, - "gemini/gemini-3.8-flash": {"spend": 0.09, "input_cost": 0.09, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0} + "gemini/gemini-3.1-pro": { + "spend": 0.00368, + "input_cost": 0.00368, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.00092, + "input_cost": 0.00092, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + }, + "gemini-3.1-pro": { + "spend": 0.003864, + "input_cost": 0.003864, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + }, + "gemini-3.8-flash": { + "spend": 0.0009568, + "input_cost": 0.0009568, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } } }, { "name": "stream_prompt_blocked", - "usage": {"fresh_input_tokens": 1000, "output_tokens": 0}, + "family": "transport", + "usage": { + "fresh_input_tokens": 1840 + }, "stream": true, "terminal": "prompt_blocked", + "expected": { + "gemini/gemini-3.1-pro": { + "spend": 0.00368, + "input_cost": 0.00368, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.00092, + "input_cost": 0.00092, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + }, + "gemini-3.1-pro": { + "spend": 0.003864, + "input_cost": 0.003864, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + }, + "gemini-3.8-flash": { + "spend": 0.0009568, + "input_cost": 0.0009568, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + } + }, + { + "name": "response_model_override", + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, "response_model_override": true, "expected": { - "gemini-3.1-pro-preview": {"spend": 0.2, "input_cost": 0.2, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, - "gemini-3.8-flash": {"spend": 0.21, "input_cost": 0.21, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.08, "input_cost": 0.08, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0}, - "gemini/gemini-3.8-flash": {"spend": 0.09, "input_cost": 0.09, "output_cost": 0.0, "prompt_tokens": 1000, "completion_tokens": 0} + "gpt-5.6": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { - "name": "all_components_chat", - "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 10, "output_tokens": 25, "reasoning_tokens": 15, "audio_input_tokens": 5, "audio_output_tokens": 3}, - "expected": { - "azure/gpt-5.4-mini": {"spend": 0.0576, "input_cost": 0.03424, "output_cost": 0.02336, "prompt_tokens": 155, "completion_tokens": 43}, - "azure/gpt-5.6": {"spend": 0.054, "input_cost": 0.0321, "output_cost": 0.0219, "prompt_tokens": 155, "completion_tokens": 43}, - "gpt-5.4-mini": {"spend": 0.0144, "input_cost": 0.00856, "output_cost": 0.00584, "prompt_tokens": 155, "completion_tokens": 43}, - "gpt-5.6": {"spend": 0.0036, "input_cost": 0.00214, "output_cost": 0.00146, "prompt_tokens": 155, "completion_tokens": 43}, - "together_ai/moonshotai/Kimi-K3": {"spend": 0.036, "input_cost": 0.0214, "output_cost": 0.0146, "prompt_tokens": 155, "completion_tokens": 43}, - "together_ai/zai-org/GLM-5.3": {"spend": 0.0396, "input_cost": 0.02354, "output_cost": 0.01606, "prompt_tokens": 155, "completion_tokens": 43} - } - }, - { - "name": "all_components_fireworks", - "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25}, - "expected": { - "fireworks_ai/deepseek-v4p1-flash": {"spend": 0.01876, "input_cost": 0.01176, "output_cost": 0.007, "prompt_tokens": 120, "completion_tokens": 25}, - "fireworks_ai/kimi-k3": {"spend": 0.01608, "input_cost": 0.01008, "output_cost": 0.006, "prompt_tokens": 120, "completion_tokens": 25}, - "fireworks_ai/qwen3p8-max": {"spend": 0.01742, "input_cost": 0.01092, "output_cost": 0.0065, "prompt_tokens": 120, "completion_tokens": 25} - } - }, - { - "name": "all_components_anthropic", - "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 10, "output_tokens": 25}, - "expected": { - "anthropic.claude-sonnet-5-v1:0": {"spend": 0.03978, "input_cost": 0.03128, "output_cost": 0.0085, "prompt_tokens": 150, "completion_tokens": 25}, - "claude-haiku-4-5": {"spend": 0.01638, "input_cost": 0.01288, "output_cost": 0.0035, "prompt_tokens": 150, "completion_tokens": 25}, - "claude-opus-5": {"spend": 0.0117, "input_cost": 0.0092, "output_cost": 0.0025, "prompt_tokens": 150, "completion_tokens": 25}, - "claude-sonnet-5": {"spend": 0.01404, "input_cost": 0.01104, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 25}, - "meta.llama4-maverick-17b-instruct-v1:0": {"spend": 0.038, "input_cost": 0.0285, "output_cost": 0.0095, "prompt_tokens": 150, "completion_tokens": 25}, - "us.anthropic.claude-opus-5-v1:0": {"spend": 0.04212, "input_cost": 0.03312, "output_cost": 0.009, "prompt_tokens": 150, "completion_tokens": 25} - } - }, - { - "name": "all_components_anthropic_stream", - "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "cache_write_5m_tokens": 20, "cache_write_1h_tokens": 10, "output_tokens": 25}, + "name": "stream_response_model_override", + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "response_model_override": true, "stream": true, "expected": { - "claude-haiku-4-5": {"spend": 0.01638, "input_cost": 0.01288, "output_cost": 0.0035, "prompt_tokens": 150, "completion_tokens": 25}, - "claude-opus-5": {"spend": 0.0117, "input_cost": 0.0092, "output_cost": 0.0025, "prompt_tokens": 150, "completion_tokens": 25}, - "claude-sonnet-5": {"spend": 0.01404, "input_cost": 0.01104, "output_cost": 0.003, "prompt_tokens": 150, "completion_tokens": 25} + "gpt-5.6": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { - "name": "all_components_gemini", - "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15, "audio_input_tokens": 5, "audio_output_tokens": 3}, + "name": "tool_call", + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "tool_call": true, "expected": { - "gemini-3.1-pro-preview": {"spend": 0.0546, "input_cost": 0.02394, "output_cost": 0.03066, "prompt_tokens": 125, "completion_tokens": 43}, - "gemini-3.8-flash": {"spend": 0.052, "input_cost": 0.0228, "output_cost": 0.0292, "prompt_tokens": 125, "completion_tokens": 43}, - "gemini/gemini-3.1-pro-preview": {"spend": 0.0234, "input_cost": 0.01026, "output_cost": 0.01314, "prompt_tokens": 125, "completion_tokens": 43}, - "gemini/gemini-3.8-flash": {"spend": 0.0208, "input_cost": 0.00912, "output_cost": 0.01168, "prompt_tokens": 125, "completion_tokens": 43} + "gpt-5.6": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } } }, { - "name": "all_components_responses", - "usage": {"fresh_input_tokens": 80, "cache_read_tokens": 40, "output_tokens": 25, "reasoning_tokens": 15}, + "name": "stream_tool_call", + "family": "transport", + "usage": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "stream": true, + "tool_call": true, "expected": { - "gpt-5.3-codex": {"spend": 0.00627, "input_cost": 0.00252, "output_cost": 0.00375, "prompt_tokens": 120, "completion_tokens": 40}, - "gpt-5.5-pro": {"spend": 0.00418, "input_cost": 0.00168, "output_cost": 0.0025, "prompt_tokens": 120, "completion_tokens": 40} + "gpt-5.6": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.4-mini": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.6": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "azure/gpt-5.4-mini": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.3-codex": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gpt-5.5-pro": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-opus-5": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.1-pro": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "gemini-3.8-flash": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + } + }, + { + "name": "stream_full_usage", + "family": "transport", + "usage": {}, + "stream": true, + "usage_by_model": { + "gpt-5.6": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330, + "audio_output_tokens": 280 + }, + "gpt-5.4-mini": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330, + "audio_output_tokens": 280 + }, + "azure/gpt-5.6": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330, + "audio_output_tokens": 280 + }, + "azure/gpt-5.4-mini": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330, + "audio_output_tokens": 280 + }, + "gpt-5.3-codex": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900 + }, + "gpt-5.5-pro": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900 + }, + "claude-opus-5": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 1024 + }, + "claude-sonnet-5": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 1024 + }, + "claude-haiku-4-5": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 1024 + }, + "us.anthropic.claude-opus-5-v1:0": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 1024 + }, + "anthropic.claude-sonnet-5-v1:0": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 1024 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "cache_write_5m_tokens": 2048, + "cache_write_1h_tokens": 1024 + }, + "gemini/gemini-3.1-pro": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330 + }, + "gemini/gemini-3.8-flash": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330, + "audio_output_tokens": 280 + }, + "gemini-3.1-pro": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330 + }, + "gemini-3.8-flash": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144, + "reasoning_tokens": 900, + "audio_input_tokens": 330, + "audio_output_tokens": 280 + }, + "together_ai/moonshotai/Kimi-K3": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "fresh_input_tokens": 1840, + "output_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "fresh_input_tokens": 1840, + "output_tokens": 412, + "cache_read_tokens": 6144 + } + }, + "expected": { + "gpt-5.6": { + "spend": 0.0600632, + "input_cost": 0.0174952, + "output_cost": 0.042568, + "prompt_tokens": 8314, + "completion_tokens": 1592 + }, + "gpt-5.4-mini": { + "spend": 0.01379264, + "input_cost": 0.00415904, + "output_cost": 0.0096336, + "prompt_tokens": 8314, + "completion_tokens": 1592 + }, + "azure/gpt-5.6": { + "spend": 0.06169072, + "input_cost": 0.01794792, + "output_cost": 0.0437428, + "prompt_tokens": 8314, + "completion_tokens": 1592 + }, + "azure/gpt-5.4-mini": { + "spend": 0.014385144, + "input_cost": 0.004348584, + "output_cost": 0.01003656, + "prompt_tokens": 8314, + "completion_tokens": 1592 + }, + "gpt-5.3-codex": { + "spend": 0.0203256, + "input_cost": 0.0036816, + "output_cost": 0.016644, + "prompt_tokens": 7984, + "completion_tokens": 1312 + }, + "gpt-5.5-pro": { + "spend": 0.203256, + "input_cost": 0.036816, + "output_cost": 0.16644, + "prompt_tokens": 7984, + "completion_tokens": 1312 + }, + "claude-opus-5": { + "spend": 0.045612, + "input_cost": 0.035312, + "output_cost": 0.0103, + "prompt_tokens": 11056, + "completion_tokens": 412 + }, + "claude-sonnet-5": { + "spend": 0.0273672, + "input_cost": 0.0211872, + "output_cost": 0.00618, + "prompt_tokens": 11056, + "completion_tokens": 412 + }, + "claude-haiku-4-5": { + "spend": 0.0091224, + "input_cost": 0.0070624, + "output_cost": 0.00206, + "prompt_tokens": 11056, + "completion_tokens": 412 + }, + "us.anthropic.claude-opus-5-v1:0": { + "spend": 0.0501732, + "input_cost": 0.0388432, + "output_cost": 0.01133, + "prompt_tokens": 11056, + "completion_tokens": 412 + }, + "anthropic.claude-sonnet-5-v1:0": { + "spend": 0.03010392, + "input_cost": 0.02330592, + "output_cost": 0.006798, + "prompt_tokens": 11056, + "completion_tokens": 412 + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "spend": 0.00305308, + "input_cost": 0.00265344, + "output_cost": 0.00039964, + "prompt_tokens": 11056, + "completion_tokens": 412 + }, + "gemini/gemini-3.1-pro": { + "spend": 0.0224108, + "input_cost": 0.0057668, + "output_cost": 0.016644, + "prompt_tokens": 8314, + "completion_tokens": 1312 + }, + "gemini/gemini-3.8-flash": { + "spend": 0.0076232, + "input_cost": 0.0015572, + "output_cost": 0.006066, + "prompt_tokens": 8314, + "completion_tokens": 1592 + }, + "gemini-3.1-pro": { + "spend": 0.02338644, + "input_cost": 0.00604524, + "output_cost": 0.0173412, + "prompt_tokens": 8314, + "completion_tokens": 1312 + }, + "gemini-3.8-flash": { + "spend": 0.007460128, + "input_cost": 0.001619488, + "output_cost": 0.00584064, + "prompt_tokens": 8314, + "completion_tokens": 1592 + }, + "together_ai/moonshotai/Kimi-K3": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "together_ai/zai-org/GLM-5.3": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "spend": 0.00250264, + "input_cost": 0.00147264, + "output_cost": 0.00103, + "prompt_tokens": 7984, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "spend": 0.00369216, + "input_cost": 0.00220896, + "output_cost": 0.0014832, + "prompt_tokens": 7984, + "completion_tokens": 412 + } } } ] diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py index 1473edb119b..e735de40027 100644 --- a/tests/e2e/cost_calculation/conftest.py +++ b/tests/e2e/cost_calculation/conftest.py @@ -7,6 +7,11 @@ deployment under test, and the request shapes plus asserted goldens live in scripted-provider sidecar (``scripted_provider.py``), registered per scenario over its control API. +The proxy must also run with ``MODEL_COST_MAP_MIN_MODEL_COUNT=1`` and +``MODEL_COST_MAP_MAX_SHRINK_RATIO=0``: the 21-entry test map trips the +fetched-cost-map integrity check (too few models, large shrink versus the +bundled map) at those env vars' defaults. + Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). """ diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/e2e/cost_calculation/cost_matrix.py index 7999d827060..5e652421182 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/e2e/cost_calculation/cost_matrix.py @@ -13,9 +13,12 @@ Two data files drive the suite; nothing in Python lists models or cases: from __future__ import annotations import base64 +import io import json +import math import random import struct +import wave import zlib from collections.abc import Mapping from dataclasses import dataclass @@ -37,22 +40,41 @@ class SearchContextCostPerQuery(BaseModel): search_context_size_high: float | None = None +class ProviderSpecificEntry(BaseModel): + """Provider-specific key rates, keyed by the named suffix litellm looks up + (``fast`` for Anthropic fast mode, ``us`` for US inference geography).""" + + model_config = ConfigDict(frozen=True) + + fast: float | None = None + us: float | None = None + + class CostMapEntry(BaseModel): """The pricing fields of a cost-map entry the matrix reads. Shaped like a - ``model_prices_and_context_window.json`` entry; unmodelled keys are ignored.""" + ``model_prices_and_context_window.json`` entry; the file is test-owned so + undeclared keys are forbidden rather than ignored.""" - model_config = ConfigDict(frozen=True, extra="ignore") + model_config = ConfigDict(frozen=True, extra="forbid") litellm_provider: str mode: str + max_tokens: int | None = None + max_input_tokens: int | None = None + max_output_tokens: int | None = None + supports_function_calling: bool | None = None input_cost_per_token: float | None = None output_cost_per_token: float | None = None cache_read_input_token_cost: float | None = None cache_creation_input_token_cost: float | None = None cache_creation_input_token_cost_above_1hr: float | None = None + cache_read_input_token_cost_above_200k_tokens: float | None = None + cache_creation_input_token_cost_above_200k_tokens: float | None = None output_cost_per_reasoning_token: float | None = None input_cost_per_audio_token: float | None = None output_cost_per_audio_token: float | None = None + input_cost_per_image_token: float | None = None + input_cost_per_video_token: float | None = None input_cost_per_token_above_200k_tokens: float | None = None output_cost_per_token_above_200k_tokens: float | None = None input_cost_per_token_flex: float | None = None @@ -61,6 +83,70 @@ class CostMapEntry(BaseModel): output_cost_per_token_priority: float | None = None search_context_cost_per_query: SearchContextCostPerQuery | None = None web_search_billing_unit: str | None = None + google_maps_grounding_cost_per_query: float | None = None + file_search_cost_per_1k_calls: float | None = None + provider_specific_entry: ProviderSpecificEntry | None = None + + +_METADATA_FIELDS: Final = frozenset( + { + "litellm_provider", + "mode", + "max_tokens", + "max_input_tokens", + "max_output_tokens", + "supports_function_calling", + } +) +_CONTAINER_FIELDS: Final = frozenset({"search_context_cost_per_query", "provider_specific_entry"}) + + +def _submodel_rate_keys( + field: str, sub: SearchContextCostPerQuery | ProviderSpecificEntry | None +) -> tuple[str, ...]: + if sub is None: + return () + return tuple( + f"{field}.{name}" + for name in type(sub).model_fields + if getattr(sub, name) is not None + ) + + +def _entry_rate_keys(entry: CostMapEntry) -> frozenset[str]: + """Every cost key an entry carries, with container subfields expanded to + dotted names (``search_context_cost_per_query.search_context_size_low``). + ``web_search_billing_unit`` counts as a rate key whenever present, + for both ``per_query`` and ``per_prompt`` values.""" + plain: Final = frozenset( + name + for name in CostMapEntry.model_fields + if name not in _METADATA_FIELDS + and name not in _CONTAINER_FIELDS + and getattr(entry, name) is not None + ) + return ( + plain + | frozenset( + _submodel_rate_keys("search_context_cost_per_query", entry.search_context_cost_per_query) + ) + | frozenset(_submodel_rate_keys("provider_specific_entry", entry.provider_specific_entry)) + ) + + +def _entry_has_rate_key(entry: CostMapEntry, rate_key: str) -> bool: + outer, _, inner = rate_key.partition(".") + if outer == "search_context_cost_per_query": + return f"{outer}.{inner}" in _submodel_rate_keys(outer, entry.search_context_cost_per_query) + if outer == "provider_specific_entry": + return f"{outer}.{inner}" in _submodel_rate_keys(outer, entry.provider_specific_entry) + value: Final[object] = getattr(entry, outer, None) + return value is not None + + +SERVICE_TIER_REQUEST_WIRES: Final = frozenset( + {"openai_chat", "azure_chat", "openai_responses", "bedrock_converse"} +) COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) @@ -94,22 +180,42 @@ class ExpectedCell(BaseModel): class Case(BaseModel): - """One request/response shape from cases.json. An exact-spend case names - its models implicitly by carrying one ``expected`` golden per map key; a - recount case (``exact_spend=False``) names them in ``models`` instead.""" + """One request/response shape from cases.json. + + ``family`` splits the matrix: ``pricing`` cases own cost keys (``owns``, + dotted subfield names allowed) or declare which keys they deliberately + leave absent (``fallback_for``) so every cost key in the map has exactly + one owning case; ``transport`` cases exercise counting/transport only and + run wherever they list membership. An exact-spend case names its models + implicitly by carrying one ``expected`` golden per map key; a recount + case (``exact_spend=False``) names them in ``models`` instead. The + feature flags drive request realism in ``_chat_body``.""" model_config = ConfigDict(frozen=True) name: str + family: Literal["pricing", "transport"] usage: ScriptedUsage + usage_by_model: Mapping[str, ScriptedUsage] = Field(default_factory=lambda: MappingProxyType({})) stream: bool = False stream_usage: Literal["final_chunk", "absent"] = "final_chunk" service_tier: Literal["flex", "priority"] | None = None + speed: Literal["fast"] | None = None + inference_geo: Literal["us"] | None = None response_model_override: bool = False exact_spend: bool = True tool_call: bool = False image_input: bool = False + audio_input: bool = False + audio_output: bool = False + video_input: bool = False + reasoning: bool = False + web_search: Literal["low", "medium", "high"] | None = None + google_maps: bool = False + file_search: bool = False terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" + owns: tuple[str, ...] = () + fallback_for: tuple[str, ...] = () expected: Mapping[str, ExpectedCell] = Field(default_factory=lambda: MappingProxyType({})) models: tuple[str, ...] = () @@ -121,11 +227,14 @@ class Case(BaseModel): def expected_for(self, model: FrontierModel) -> ExpectedCell: return self.expected[model.map_key] + def usage_for(self, map_key: str) -> ScriptedUsage: + return self.usage_by_model.get(map_key, self.usage) + def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( scenario_id=scenario_id, wire=model.wire, - usage=self.usage, + usage=self.usage_for(model.map_key), model=model.provider_model, output=ScriptedOutput( text=text, @@ -137,6 +246,8 @@ class Case(BaseModel): ), stream_usage=self.stream_usage, service_tier=self.service_tier, + speed=self.speed, + inference_geo=self.inference_geo, ) @@ -183,7 +294,7 @@ _PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = MappingProx { ("openai", "chat"): _ProviderWiring("openai_chat", "openai", MappingProxyType({})), ("openai", "responses"): _ProviderWiring( - "openai_responses", "openai", MappingProxyType({}) + "openai_responses", "openai/responses", MappingProxyType({}) ), ("anthropic", "chat"): _ProviderWiring( "anthropic_messages", "anthropic", MappingProxyType({}) @@ -226,7 +337,13 @@ class FrontierModel: @property def override_rates(self) -> CostMapEntry: - if self.base_model is not None or self.override_map_key is None: + # bedrock_converse responses carry no model field, so a reported-model + # override can never repoint pricing there, same as a base_model pin. + if ( + self.base_model is not None + or self.wire == "bedrock_converse" + or self.override_map_key is None + ): return self.rates return COST_MAP[self.override_map_key] @@ -338,6 +455,31 @@ def _png_chunk(tag: bytes, payload: bytes) -> bytes: return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", zlib.crc32(tag + payload)) +def audio_input_data_url() -> str: + """A deterministic 0.5 s 16-bit PCM WAV (8 kHz, 220 Hz sine) as a data + URL, small enough to stay a fixture but real audio to the provider.""" + frames: Final = b"".join( + struct.pack(" str: + """A deterministic mp4-looking blob (ftyp box plus a fixed mdat payload) + as a data URL; only the media type and bytes matter to the wire.""" + ftyp: Final = struct.pack(">I4s4sI4s4s", 24, b"ftyp", b"isom", 0x200, b"isom", b"iso6") + mdat_payload: Final = bytes((i * 7 + 13) % 256 for i in range(4096)) + mdat: Final = struct.pack(">I4s", 8 + len(mdat_payload), b"mdat") + mdat_payload + return "data:video/mp4;base64," + base64.b64encode(ftyp + mdat).decode() + + def image_input_data_url() -> str: """A deterministic 256x256 RGB noise PNG as a data URL; noise compresses poorly on purpose so the base64 payload stays well above 100 KB and would @@ -357,6 +499,8 @@ def image_input_data_url() -> str: IMAGE_INPUT_DATA_URL: Final = image_input_data_url() +AUDIO_INPUT_DATA_URL: Final = audio_input_data_url() +VIDEO_INPUT_DATA_URL: Final = video_input_data_url() def matrix_data_errors() -> tuple[str, ...]: @@ -381,6 +525,48 @@ def matrix_data_errors() -> tuple[str, ...]: for case in CASES if case.exact_spend == bool(case.models) or case.exact_spend != bool(case.expected) ) + all_pairs: Final = frozenset( + (map_key, key) + for map_key, entry in COST_MAP.items() + for key in _entry_rate_keys(entry) + ) + owned_pairs: Final = tuple( + (map_key, key) + for case in CASES + if case.family == "pricing" + for map_key in case.expected + for key in case.owns + if map_key in COST_MAP and _entry_has_rate_key(COST_MAP[map_key], key) + ) + unowned_pairs: Final = sorted( + f"{map_key}:{key}" for map_key, key in all_pairs - frozenset(owned_pairs) + ) + duplicate_pairs: Final = sorted( + f"{map_key}:{key}" + for map_key, key in set(owned_pairs) + if owned_pairs.count((map_key, key)) > 1 + ) + owns_without_holder: Final = sorted( + f"{case.name}:{key}" + for case in CASES + for key in case.owns + if not any( + map_key in COST_MAP and _entry_has_rate_key(COST_MAP[map_key], key) + for map_key in case.expected + ) + ) + fallback_violations: Final = sorted( + f"{case.name}:{map_key}:{key}" + for case in CASES + for key in case.fallback_for + for map_key in (*case.expected, *case.models) + if map_key in COST_MAP and _entry_has_rate_key(COST_MAP[map_key], key) + ) + family_violations: Final = sorted( + case.name + for case in CASES + if (case.family == "transport") != (not case.owns and not case.fallback_for) + ) input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) findings: Final = ( ( @@ -404,5 +590,30 @@ def matrix_data_errors() -> tuple[str, ...]: if len(input_rates) != len(set(input_rates)) else None ), + ( + f"(model, rate key) pairs with no owning case: {unowned_pairs}" + if unowned_pairs + else None + ), + ( + f"(model, rate key) pairs owned by more than one case: {duplicate_pairs}" + if duplicate_pairs + else None + ), + ( + f"owns keys absent on all of the case's expected models: {owns_without_holder}" + if owns_without_holder + else None + ), + ( + f"fallback_for keys a case's models actually carry: {fallback_violations}" + if fallback_violations + else None + ), + ( + f"cases with owns/fallback_for inconsistent with family: {family_violations}" + if family_violations + else None + ), ) return tuple(finding for finding in findings if finding is not None) diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/e2e/cost_calculation/scripted_provider.py index 90d95441e5c..c154dcdae62 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/e2e/cost_calculation/scripted_provider.py @@ -87,6 +87,56 @@ _TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( ) +_BASE_USAGE_FIELDS: Final = frozenset({"fresh_input_tokens", "output_tokens"}) +_OPENAI_FAMILY_USAGE: Final = frozenset( + { + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "web_search_calls", + } +) +_CACHE_WRITE_USAGE: Final = frozenset({"cache_write_5m_tokens", "cache_write_1h_tokens"}) +_GEMINI_USAGE: Final = frozenset( + { + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "image_input_tokens", + "video_input_tokens", + "web_search_calls", + "google_maps_calls", + } +) + +_USAGE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( + { + wire: usage + for wire, usage in ( + ("openai_chat", _OPENAI_FAMILY_USAGE), + ("azure_chat", _OPENAI_FAMILY_USAGE), + ("together_chat", _OPENAI_FAMILY_USAGE), + ("fireworks_chat", _OPENAI_FAMILY_USAGE), + ( + "openai_responses", + frozenset( + {"cache_read_tokens", "reasoning_tokens", "web_search_calls", "file_search_calls"} + ), + ), + ( + "anthropic_messages", + frozenset({"cache_read_tokens", "web_search_calls"}) | _CACHE_WRITE_USAGE, + ), + ("bedrock_converse", frozenset({"cache_read_tokens"}) | _CACHE_WRITE_USAGE), + ("gemini_generate", _GEMINI_USAGE), + ("vertex_generate", _GEMINI_USAGE), + ) + } +) + + class ScriptedToolCall(BaseModel): """A single function call the scripted output emits instead of text. ``arguments`` is the wire's JSON string (~250 chars), sliced into deltas @@ -116,7 +166,11 @@ class ScriptedUsage(BaseModel): reasoning_tokens: int = 0 audio_input_tokens: int = 0 audio_output_tokens: int = 0 + image_input_tokens: int = 0 + video_input_tokens: int = 0 web_search_calls: int = 0 + google_maps_calls: int = 0 + file_search_calls: int = 0 class ScriptedOutput(BaseModel): @@ -151,6 +205,10 @@ class Scenario(BaseModel): model: str stream_usage: StreamUsage = "final_chunk" service_tier: ServiceTier | None = None + # Anthropic fast mode and US inference geography; emitted on the anthropic + # usage object only (litellm reads them there), so they are response-side. + speed: Literal["fast"] | None = None + inference_geo: Literal["us"] | None = None @model_validator(mode="after") def _check_terminal_supported(self) -> Scenario: @@ -161,6 +219,20 @@ class Scenario(BaseModel): raise ValueError( f"wire {self.wire} cannot emit terminal={self.output.terminal}" ) + unsupported: Final = frozenset( + field + for field in self.usage.model_fields_set + if getattr(self.usage, field) + and field not in (_USAGE_CAPS.get(self.wire, frozenset()) | _BASE_USAGE_FIELDS) + ) + if unsupported: + raise ValueError( + f"wire {self.wire} cannot express usage fields {sorted(unsupported)}" + ) + if (self.speed or self.inference_geo) and self.wire != "anthropic_messages": + raise ValueError( + f"wire {self.wire} cannot emit speed/inference_geo (anthropic usage fields)" + ) return self @property @@ -215,32 +287,10 @@ def _sse(events: tuple[tuple[str | None, Mapping[str, object] | str], ...]) -> b def _openai_usage(u: ScriptedUsage) -> Mapping[str, object]: - prompt_tokens: Final = ( - u.fresh_input_tokens - + u.cache_read_tokens - + u.cache_write_5m_tokens - + u.cache_write_1h_tokens - + u.audio_input_tokens - ) + prompt_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens completion_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens prompt_details: Final = _jobj_opt( ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, - ( - ("cache_write_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) - if u.cache_write_5m_tokens or u.cache_write_1h_tokens - else None - ), - ( - ( - "cache_creation_token_details", - _jobj( - ("ephemeral_5m_input_tokens", u.cache_write_5m_tokens), - ("ephemeral_1h_input_tokens", u.cache_write_1h_tokens), - ), - ) - if u.cache_write_5m_tokens or u.cache_write_1h_tokens - else None - ), ("audio_tokens", u.audio_input_tokens) if u.audio_input_tokens else None, ) completion_details: Final = _jobj_opt( @@ -256,12 +306,16 @@ def _openai_usage(u: ScriptedUsage) -> Mapping[str, object]: ) -def _anthropic_usage(u: ScriptedUsage) -> Mapping[str, object]: +def _anthropic_usage(scenario: Scenario) -> Mapping[str, object]: # Anthropic reports uncached-only input_tokens; cache reads and writes ride # top-level fields, with the 5m/1h write split under cache_creation. + u: Final = scenario.usage return _jobj_opt( ("input_tokens", u.fresh_input_tokens), ("output_tokens", u.output_tokens), + ("service_tier", scenario.service_tier) if scenario.service_tier else None, + ("speed", scenario.speed) if scenario.speed else None, + ("inference_geo", scenario.inference_geo) if scenario.inference_geo else None, ("cache_read_input_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, ( ("cache_creation_input_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) @@ -287,18 +341,24 @@ def _anthropic_usage(u: ScriptedUsage) -> Mapping[str, object]: ) -def _gemini_usage(u: ScriptedUsage) -> Mapping[str, object]: - # promptTokenCount carries the cached count inside it; TEXT modality is the - # cached-inclusive text count so litellm's implicit-caching subtraction lands - # on the fresh figure. candidatesTokenCount includes reasoning + audio. - prompt_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - candidates: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens +def _gemini_usage(scenario: Scenario) -> Mapping[str, object]: + # Real generateContent accounting: promptTokenCount carries the cached count + # inside it (TEXT modality is the cached-inclusive text count so litellm's + # implicit-caching subtraction lands on the fresh figure), candidatesTokenCount + # excludes thoughts, thoughtsTokenCount reports them separately, and + # totalTokenCount sums all three. Image/video input ride promptTokensDetails. + u: Final = scenario.usage + prompt_tokens: Final = ( + u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens + + u.image_input_tokens + u.video_input_tokens + ) + candidates: Final = u.output_tokens + u.audio_output_tokens return _jobj_opt( ("promptTokenCount", prompt_tokens), ("candidatesTokenCount", candidates), - ("totalTokenCount", prompt_tokens + candidates), - ("cachedContentTokenCount", u.cache_read_tokens) if u.cache_read_tokens else None, ("thoughtsTokenCount", u.reasoning_tokens) if u.reasoning_tokens else None, + ("totalTokenCount", prompt_tokens + candidates + u.reasoning_tokens), + ("cachedContentTokenCount", u.cache_read_tokens) if u.cache_read_tokens else None, ( "promptTokensDetails", ( @@ -308,19 +368,66 @@ def _gemini_usage(u: ScriptedUsage) -> Mapping[str, object]: if u.audio_input_tokens else () ), + *( + (_jobj(("modality", "IMAGE"), ("tokenCount", u.image_input_tokens)),) + if u.image_input_tokens + else () + ), + *( + (_jobj(("modality", "VIDEO"), ("tokenCount", u.video_input_tokens)),) + if u.video_input_tokens + else () + ), ), ), ( ( "candidatesTokensDetails", ( - _jobj(("modality", "TEXT"), ("tokenCount", u.output_tokens + u.reasoning_tokens)), + _jobj(("modality", "TEXT"), ("tokenCount", u.output_tokens)), _jobj(("modality", "AUDIO"), ("tokenCount", u.audio_output_tokens)), ), ) if u.audio_output_tokens else None ), + ( + ( + "trafficType", + {"flex": "ON_DEMAND_FLEX", "priority": "ON_DEMAND_PRIORITY"}[ + scenario.service_tier + ], + ) + if scenario.service_tier + else None + ), + ) + + +def _gemini_grounding_metadata(scenario: Scenario) -> Mapping[str, object] | None: + """groundingMetadata for the search/Maps flags. Maps items carry maps + chunks and googleMapsWidgetContextToken so litellm bills them as Maps + queries, not web search.""" + u: Final = scenario.usage + if not u.web_search_calls and not u.google_maps_calls: + return None + if u.google_maps_calls: + return _jobj( + ( + "webSearchQueries", + tuple(f"maps query {i}" for i in range(u.google_maps_calls)), + ), + ( + "groundingChunks", + tuple( + _jobj(("maps", _jobj(("uri", f"https://maps.google.com/?cid={i}")))) + for i in range(u.google_maps_calls) + ), + ), + ("googleMapsWidgetContextToken", f"token_{scenario.scenario_id}"), + ) + return _jobj( + ("webSearchQueries", tuple(f"query {i}" for i in range(u.web_search_calls))), ) @@ -572,7 +679,7 @@ def _anthropic_body(scenario: Scenario, requested_model: str) -> Mapping[str, ob ("model", scenario.output.response_model or requested_model), ("content", _anthropic_content(scenario)), ("stop_reason", _anthropic_stop_reason(scenario)), - ("usage", _anthropic_usage(scenario.usage)), + ("usage", _anthropic_usage(scenario)), ) @@ -581,7 +688,7 @@ def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: input_usage: Final = _jobj( *( (key, value) - for key, value in _anthropic_usage(scenario.usage).items() + for key, value in _anthropic_usage(scenario).items() if key != "output_tokens" ) ) @@ -685,7 +792,7 @@ def _gemini_prompt_blocked_body(scenario: Scenario, requested_model: str) -> Map ), ), ), - ("usageMetadata", _gemini_usage(scenario.usage)), + ("usageMetadata", _gemini_usage(scenario)), ("modelVersion", scenario.output.response_model or requested_model), ) @@ -728,22 +835,14 @@ def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, objec ), ("index", 0), ( - ( - "groundingMetadata", - _jobj( - ( - "webSearchQueries", - tuple(f"query {i}" for i in range(scenario.usage.web_search_calls)), - ) - ), - ) - if scenario.usage.web_search_calls + ("groundingMetadata", _gemini_grounding_metadata(scenario)) + if _gemini_grounding_metadata(scenario) is not None else None ), ), ), ), - ("usageMetadata", _gemini_usage(scenario.usage)), + ("usageMetadata", _gemini_usage(scenario)), ("modelVersion", scenario.output.response_model or requested_model), ) @@ -762,7 +861,7 @@ def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: None, _jobj( ("candidates", ()), - ("usageMetadata", _gemini_usage(scenario.usage)), + ("usageMetadata", _gemini_usage(scenario)), ("modelVersion", scenario.output.response_model or requested_model), ), ), @@ -788,6 +887,16 @@ def _responses_output(scenario: Scenario) -> tuple[Mapping[str, object], ...]: _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) for i in range(scenario.usage.web_search_calls) ), + *( + _jobj( + ("type", "file_search_call"), + ("id", f"fs_{i}"), + ("status", "completed"), + ("queries", (f"query {i}",)), + ("results", ()), + ) + for i in range(scenario.usage.file_search_calls) + ), _jobj( ("type", "function_call"), ("id", f"fc_{scenario.scenario_id}"), @@ -853,9 +962,50 @@ def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: "response.incomplete" if scenario.output.terminal == "incomplete" else "response.completed" ) output_index: Final = ( - scenario.usage.web_search_calls + (1 if scenario.output.terminal == "unvalidated" else 0) + scenario.usage.web_search_calls + + scenario.usage.file_search_calls + + (1 if scenario.output.terminal == "unvalidated" else 0) ) - middle_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + file_search_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = tuple( + event + for i in range(scenario.usage.file_search_calls) + for event in ( + ( + "response.output_item.added", + _jobj( + ("type", "response.output_item.added"), + ("output_index", i), + ( + "item", + _jobj( + ("type", "file_search_call"), + ("id", f"fs_{i}"), + ("status", "in_progress"), + ("queries", ()), + ), + ), + ), + ), + ( + "response.output_item.done", + _jobj( + ("type", "response.output_item.done"), + ("output_index", i), + ( + "item", + _jobj( + ("type", "file_search_call"), + ("id", f"fs_{i}"), + ("status", "completed"), + ("queries", (f"query {i}",)), + ("results", ()), + ), + ), + ), + ), + ) + ) + call_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( ( ( "response.output_item.added", @@ -911,6 +1061,10 @@ def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: ), ) ) + middle_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( + *file_search_events, + *call_events, + ) return _sse( ( ("response.created", _jobj(("type", "response.created"), ("response", created))), @@ -976,7 +1130,7 @@ def _bedrock_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: - return _jobj( + return _jobj_opt( ( "output", _jobj( @@ -992,6 +1146,11 @@ def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: ("stopReason", _bedrock_stop_reason(scenario)), ("usage", _bedrock_usage(scenario.usage)), ("metrics", _jobj(("latencyMs", 42))), + ( + ("serviceTier", _jobj(("type", scenario.service_tier))) + if scenario.service_tier + else None + ), ) @@ -1083,9 +1242,14 @@ def _bedrock_eventstream(scenario: Scenario) -> bytes: ( _aws_event_frame( "metadata", - _jobj( + _jobj_opt( ("usage", _bedrock_usage(scenario.usage)), ("metrics", _jobj(("latencyMs", 42))), + ( + ("serviceTier", _jobj(("type", scenario.service_tier))) + if scenario.service_tier + else None + ), ), ), ) diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py index 03dab6be5e7..004cb4d839e 100644 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ b/tests/e2e/cost_calculation/test_token_pricing_e2e.py @@ -16,8 +16,11 @@ from typing import Final from conftest import CostCalcClient, cost_rows, register_scenario_deployment from cost_matrix import ( + AUDIO_INPUT_DATA_URL, FRONTIER_MODELS, IMAGE_INPUT_DATA_URL, + SERVICE_TIER_REQUEST_WIRES, + VIDEO_INPUT_DATA_URL, Case, FrontierModel, cases_for, @@ -27,15 +30,27 @@ from cost_matrix import ( from e2e_config import unique_marker from lifecycle import ResourceManager from models import ( + CacheControl, + ChatAudio, ChatBody, ChatMessage, ChatStreamOptions, ChatTool, ChatToolFunction, + FileContentPart, + FileObject, + FileSearchTool, + GoogleMapsTool, + GoogleSearchTool, + HostedWebSearchTool, ImageContentPart, ImageUrl, + InputAudio, + InputAudioContentPart, TextContentPart, + WebSearchOptions, ) +from scripted_provider import ScriptedUsage, Wire pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark @@ -52,40 +67,131 @@ def _case_id(param: tuple[FrontierModel, Case]) -> str: return f"{model.map_key.replace('/', '-')}-{case.name}" -def _chat_body(model_name: str, marker: str, case: Case) -> ChatBody: - return ChatBody( - model=model_name, - messages=( - ChatMessage( - role="user", - content=( - [ - TextContentPart(text=f"{marker} scripted pricing call"), - ImageContentPart(image_url=ImageUrl(url=IMAGE_INPUT_DATA_URL)), - ] - if case.image_input - else f"{marker} scripted pricing call" - ), - ), +_CACHE_WIRES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) +_WEB_SEARCH_OPTION_WIRES: Final = frozenset({"openai_chat", "azure_chat", "openai_responses"}) + + +def _cache_control(usage: ScriptedUsage, wire: Wire) -> CacheControl | None: + if wire not in _CACHE_WIRES: + return None + if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens): + return None + return CacheControl(type="ephemeral", ttl="1h" if usage.cache_write_1h_tokens else None) + + +def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) -> ChatBody: + usage: Final = case.usage_for(model.map_key) + user_parts: Final = ( + TextContentPart( + text=f"{marker} summarize the attached material in one line and name the city weather", ), - stream=case.stream, - stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, - service_tier=case.service_tier, - tools=( + *( + (ImageContentPart(image_url=ImageUrl(url=IMAGE_INPUT_DATA_URL, detail="high")),) + if case.image_input + else () + ), + *( + ( + InputAudioContentPart( + input_audio=InputAudio(data=AUDIO_INPUT_DATA_URL.split(",", 1)[1], format="wav") + ), + ) + if case.audio_input + else () + ), + *( + (FileContentPart(file=FileObject(file_data=VIDEO_INPUT_DATA_URL, format="mp4")),) + if case.video_input + else () + ), + ) + tools: Final = ( + *( ( ChatTool( function=ChatToolFunction( name="get_weather", + description="Get the current weather and a short forecast for a city.", parameters={ "type": "object", - "properties": {"city": {"type": "string"}}, + "properties": { + "city": {"type": "string", "description": "City name"}, + "days": {"type": "integer", "description": "Forecast horizon in days"}, + "units": {"type": "string", "enum": ["metric", "imperial"]}, + }, + "required": ["city"], }, ) ), ) if case.tool_call + else () + ), + *( + (HostedWebSearchTool(type="web_search_20250305", name="web_search", max_uses=5),) + if case.web_search is not None and model.wire == "anthropic_messages" + else () + ), + *( + (GoogleSearchTool(),) + if case.web_search is not None and model.wire in ("gemini_generate", "vertex_generate") + else () + ), + *((GoogleMapsTool(),) if case.google_maps else ()), + *((FileSearchTool(vector_store_ids=["vs_cost_calc_fixture"]),) if case.file_search else ()), + ) + return ChatBody( + model=model_name, + messages=( + ChatMessage( + role="system", + content=[ + TextContentPart( + text=( + "You are a deterministic pricing-harness assistant. " + "Keep answers to a single short line." + ), + cache_control=_cache_control(usage, model.wire), + ) + ], + ), + ChatMessage(role="user", content=list(user_parts)), + ), + stream=case.stream, + stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, + service_tier=( + case.service_tier + if case.service_tier is not None and model.wire in SERVICE_TIER_REQUEST_WIRES else None ), + reasoning_effort="medium" if case.reasoning else None, + modalities=( + ["text"] if case.audio_input else (["text", "audio"] if case.audio_output else None) + ), + audio=( + ChatAudio(voice="alloy", format="pcm16") if case.audio_output else None + ), + web_search_options=( + WebSearchOptions(search_context_size=case.web_search) + if case.web_search is not None and model.wire in _WEB_SEARCH_OPTION_WIRES + else None + ), + tools=tools or None, + tool_choice="auto" if case.tool_call and model.wire != "bedrock_converse" else None, + # The test-owned cost map carries no supports_* flags, so litellm's + # optional-params gate rejects the realistic request fields; allowlist + # exactly the ones this case sends. + allowed_openai_params=[ + name + for name, sent in ( + ("tool_choice", case.tool_call and model.wire != "bedrock_converse"), + ("modalities", case.audio_input or case.audio_output), + ("audio", case.audio_output), + ("web_search_options", case.web_search is not None), + ("reasoning_effort", case.reasoning), + ) + if sent + ], ) @@ -105,7 +211,7 @@ class TestTokenPricing: response: Final = client.proxy.transport.send( "/chat/completions", headers=client.proxy.transport.bearer(scoped_key), - json=_chat_body(model_name, marker, case), + json=_chat_body(model, case, model_name, marker), stream=case.stream, ) assert response.ok, ( diff --git a/tests/e2e/cost_map.json b/tests/e2e/cost_map.json index 85cd5ade3d5..117e9b33636 100644 --- a/tests/e2e/cost_map.json +++ b/tests/e2e/cost_map.json @@ -1,525 +1,411 @@ { - "anthropic.claude-sonnet-5-v1:0": { - "cache_creation_input_token_cost": 0.00051, - "cache_creation_input_token_cost_above_1hr": 0.00068, - "cache_read_input_token_cost": 1.7e-05, - "input_cost_per_token": 0.00017, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "gpt-5.6": { + "cache_read_input_token_cost": 1.75e-07, + "input_cost_per_audio_token": 4e-05, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_flex": 8.75e-07, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 0.00034, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true - }, - "azure/gpt-5.4-mini": { - "cache_creation_input_token_cost": 0.00048, - "cache_creation_input_token_cost_above_1hr": 0.00064, - "cache_read_input_token_cost": 1.6e-05, - "input_cost_per_audio_token": 0.00096, - "input_cost_per_token": 0.00016, - "input_cost_per_token_above_200k_tokens": 0.00128, - "input_cost_per_token_flex": 0.00024, - "input_cost_per_token_priority": 0.000272, - "litellm_provider": "azure", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.00112, - "output_cost_per_reasoning_token": 0.0008, - "output_cost_per_token": 0.00032, - "output_cost_per_token_above_200k_tokens": 0.00144, - "output_cost_per_token_flex": 0.0004, - "output_cost_per_token_priority": 0.000432, + "output_cost_per_audio_token": 8e-05, + "output_cost_per_reasoning_token": 1.6e-05, + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_flex": 7e-06, + "output_cost_per_token_priority": 2.8e-05, "search_context_cost_per_query": { - "search_context_size_high": 0.03, "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "supports_function_calling": true + }, + "gpt-5.4-mini": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 3.5e-07, + "input_cost_per_token_flex": 1.75e-07, + "input_cost_per_token_priority": 7e-07, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_reasoning_token": 3.2e-06, + "output_cost_per_token": 2.8e-06, + "output_cost_per_token_flex": 1.4e-06, + "output_cost_per_token_priority": 5.6e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true }, "azure/gpt-5.6": { - "cache_creation_input_token_cost": 0.00045, - "cache_creation_input_token_cost_above_1hr": 0.0006, - "cache_read_input_token_cost": 1.5e-05, - "input_cost_per_audio_token": 0.0009, - "input_cost_per_token": 0.00015, - "input_cost_per_token_above_200k_tokens": 0.0012, - "input_cost_per_token_flex": 0.000225, - "input_cost_per_token_priority": 0.000255, + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_audio_token": 4.1e-05, + "input_cost_per_token": 1.8e-06, + "input_cost_per_token_flex": 9e-07, + "input_cost_per_token_priority": 3.6e-06, "litellm_provider": "azure", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00105, - "output_cost_per_reasoning_token": 0.00075, - "output_cost_per_token": 0.0003, - "output_cost_per_token_above_200k_tokens": 0.00135, - "output_cost_per_token_flex": 0.000375, - "output_cost_per_token_priority": 0.000405, + "output_cost_per_audio_token": 8.2e-05, + "output_cost_per_reasoning_token": 1.65e-05, + "output_cost_per_token": 1.44e-05, + "output_cost_per_token_flex": 7.2e-06, + "output_cost_per_token_priority": 2.88e-05, "search_context_cost_per_query": { - "search_context_size_high": 0.03, "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "supports_function_calling": true }, - "claude-haiku-4-5": { - "cache_creation_input_token_cost": 0.00021, - "cache_creation_input_token_cost_above_1hr": 0.00028000000000000003, - "cache_read_input_token_cost": 7e-06, - "input_cost_per_token": 7.000000000000001e-05, - "litellm_provider": "anthropic", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "azure/gpt-5.4-mini": { + "cache_read_input_token_cost": 3.6e-08, + "input_cost_per_audio_token": 1.05e-05, + "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 7.2e-07, + "litellm_provider": "azure", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 0.00014000000000000001, + "output_cost_per_audio_token": 2.1e-05, + "output_cost_per_reasoning_token": 3.3e-06, + "output_cost_per_token": 2.88e-06, + "output_cost_per_token_flex": 1.44e-06, + "output_cost_per_token_priority": 5.76e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.03, "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "supports_function_calling": true + }, + "gpt-5.3-codex": { + "cache_read_input_token_cost": 1.5e-07, + "file_search_cost_per_1k_calls": 0.0025, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 3e-06, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "responses", + "output_cost_per_reasoning_token": 1.3e-05, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_flex": 6e-06, + "output_cost_per_token_priority": 2.4e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "gpt-5.5-pro": { + "cache_read_input_token_cost": 1.5e-06, + "file_search_cost_per_1k_calls": 0.0025, + "input_cost_per_token": 1.5e-05, + "input_cost_per_token_flex": 7.5e-06, + "input_cost_per_token_priority": 3e-05, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "responses", + "output_cost_per_reasoning_token": 0.00013, + "output_cost_per_token": 0.00012, + "output_cost_per_token_flex": 6e-05, + "output_cost_per_token_priority": 0.00024, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true }, "claude-opus-5": { - "cache_creation_input_token_cost": 0.00015000000000000001, - "cache_creation_input_token_cost_above_1hr": 0.0002, - "cache_read_input_token_cost": 4.9999999999999996e-06, - "input_cost_per_token": 5e-05, + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_200k_tokens": 1e-05, + "input_cost_per_token_priority": 6.25e-06, "litellm_provider": "anthropic", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 0.0001, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "output_cost_per_token_priority": 3.125e-05, + "provider_specific_entry": { + "fast": 6.0, + "us": 1.1 }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "search_context_cost_per_query": { + "search_context_size_medium": 0.01 + }, + "supports_function_calling": true }, "claude-sonnet-5": { - "cache_creation_input_token_cost": 0.00018, - "cache_creation_input_token_cost_above_1hr": 0.00024000000000000003, - "cache_read_input_token_cost": 6e-06, - "input_cost_per_token": 6.000000000000001e-05, + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "input_cost_per_token_priority": 3.75e-06, "litellm_provider": "anthropic", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 0.00012000000000000002, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "output_cost_per_token_priority": 1.875e-05, + "provider_specific_entry": { + "us": 1.1 }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "search_context_cost_per_query": { + "search_context_size_medium": 0.01 + }, + "supports_function_calling": true }, - "fireworks_ai/deepseek-v4p1-flash": { - "cache_creation_input_token_cost": 0.00033, - "cache_creation_input_token_cost_above_1hr": 0.00044, - "cache_read_input_token_cost": 1.4e-05, - "input_cost_per_audio_token": 0.00066, - "input_cost_per_token": 0.00014000000000000001, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_priority": 1.25e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00077, - "output_cost_per_reasoning_token": 0.00055, - "output_cost_per_token": 0.00028000000000000003, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "output_cost_per_token": 5e-06, + "output_cost_per_token_priority": 6.25e-06, + "provider_specific_entry": { + "us": 1.1 }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "search_context_cost_per_query": { + "search_context_size_medium": 0.01 + }, + "supports_function_calling": true }, - "fireworks_ai/kimi-k3": { - "cache_creation_input_token_cost": 0.00033, - "cache_creation_input_token_cost_above_1hr": 0.00044, - "cache_read_input_token_cost": 1.2e-05, - "input_cost_per_audio_token": 0.00066, - "input_cost_per_token": 0.00012000000000000002, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "us.anthropic.claude-opus-5-v1:0": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_200k_tokens": 1.1e-05, + "input_cost_per_token_flex": 2.75e-06, + "input_cost_per_token_priority": 6.875e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00077, - "output_cost_per_reasoning_token": 0.00055, - "output_cost_per_token": 0.00024000000000000003, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "output_cost_per_token": 2.75e-05, + "output_cost_per_token_above_200k_tokens": 4.125e-05, + "output_cost_per_token_flex": 1.375e-05, + "output_cost_per_token_priority": 3.4375e-05, + "supports_function_calling": true }, - "fireworks_ai/qwen3p8-max": { - "cache_creation_input_token_cost": 0.00033, - "cache_creation_input_token_cost_above_1hr": 0.00044, - "cache_read_input_token_cost": 1.3e-05, - "input_cost_per_audio_token": 0.00066, - "input_cost_per_token": 0.00013000000000000002, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "anthropic.claude-sonnet-5-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "input_cost_per_token_flex": 1.65e-06, + "input_cost_per_token_priority": 4.125e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00077, - "output_cost_per_reasoning_token": 0.00055, - "output_cost_per_token": 0.00026000000000000003, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_flex": 8.25e-06, + "output_cost_per_token_priority": 2.0625e-05, + "supports_function_calling": true }, - "gemini-3.1-pro-preview": { - "cache_read_input_token_cost": 2.1e-05, - "input_cost_per_audio_token": 0.00126, - "input_cost_per_token": 0.00021, - "input_cost_per_token_above_200k_tokens": 0.00168, - "input_cost_per_token_flex": 0.000315, - "input_cost_per_token_priority": 0.000357, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "meta.llama4-maverick-17b-instruct-v1:0": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00147, - "output_cost_per_reasoning_token": 0.00105, - "output_cost_per_token": 0.00042, - "output_cost_per_token_above_200k_tokens": 0.00189, - "output_cost_per_token_flex": 0.000525, - "output_cost_per_token_priority": 0.000567, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" + "output_cost_per_token": 9.7e-07, + "supports_function_calling": true }, - "gemini-3.8-flash": { - "cache_read_input_token_cost": 2e-05, - "input_cost_per_audio_token": 0.0012, - "input_cost_per_token": 0.0002, - "input_cost_per_token_above_200k_tokens": 0.0016, - "input_cost_per_token_flex": 0.0003, - "input_cost_per_token_priority": 0.00034, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 0.0014, - "output_cost_per_reasoning_token": 0.001, - "output_cost_per_token": 0.0004, - "output_cost_per_token_above_200k_tokens": 0.0018, - "output_cost_per_token_flex": 0.0005, - "output_cost_per_token_priority": 0.00054, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, - "web_search_billing_unit": "per_query" - }, - "gemini/gemini-3.1-pro-preview": { - "cache_read_input_token_cost": 9e-06, - "input_cost_per_audio_token": 0.00054, - "input_cost_per_token": 9e-05, - "input_cost_per_token_above_200k_tokens": 0.00072, - "input_cost_per_token_flex": 0.000135, - "input_cost_per_token_priority": 0.000153, + "gemini/gemini-3.1-pro": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 2.6e-06, + "input_cost_per_image_token": 2.2e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 2.5e-06, + "input_cost_per_video_token": 2.4e-06, "litellm_provider": "gemini", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.0006299999999999999, - "output_cost_per_reasoning_token": 0.00045000000000000004, - "output_cost_per_token": 0.00018, - "output_cost_per_token_above_200k_tokens": 0.0008100000000000001, - "output_cost_per_token_flex": 0.00022500000000000002, - "output_cost_per_token_priority": 0.000243, + "output_cost_per_reasoning_token": 1.3e-05, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_flex": 6e-06, + "output_cost_per_token_priority": 1.5e-05, "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "search_context_size_medium": 0.035 }, "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, "web_search_billing_unit": "per_query" }, "gemini/gemini-3.8-flash": { - "cache_read_input_token_cost": 8e-06, - "input_cost_per_audio_token": 0.00048, - "input_cost_per_token": 8e-05, - "input_cost_per_token_above_200k_tokens": 0.00064, - "input_cost_per_token_flex": 0.00012, - "input_cost_per_token_priority": 0.000136, + "cache_read_input_token_cost": 5e-08, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_image_token": 5.5e-07, + "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, + "input_cost_per_token_priority": 6.25e-07, + "input_cost_per_video_token": 6e-07, "litellm_provider": "gemini", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00056, - "output_cost_per_reasoning_token": 0.0004, - "output_cost_per_token": 0.00016, - "output_cost_per_token_above_200k_tokens": 0.00072, - "output_cost_per_token_flex": 0.0002, - "output_cost_per_token_priority": 0.000216, + "output_cost_per_audio_token": 6e-06, + "output_cost_per_reasoning_token": 3.5e-06, + "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 3.75e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "search_context_size_medium": 0.035 + }, + "supports_function_calling": true, + "web_search_billing_unit": "per_prompt" + }, + "gemini-3.1-pro": { + "cache_read_input_token_cost": 2.1e-07, + "cache_read_input_token_cost_above_200k_tokens": 4.2e-07, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 2.7e-06, + "input_cost_per_image_token": 2.3e-06, + "input_cost_per_token": 2.1e-06, + "input_cost_per_token_above_200k_tokens": 4.2e-06, + "input_cost_per_token_flex": 1.05e-06, + "input_cost_per_token_priority": 2.625e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 1.35e-05, + "output_cost_per_token": 1.26e-05, + "output_cost_per_token_above_200k_tokens": 1.89e-05, + "output_cost_per_token_flex": 6.3e-06, + "output_cost_per_token_priority": 1.575e-05, + "search_context_cost_per_query": { + "search_context_size_medium": 0.035 }, "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true, "web_search_billing_unit": "per_query" }, - "gpt-5.3-codex": { - "cache_read_input_token_cost": 3e-06, - "input_cost_per_token": 3.0000000000000004e-05, - "input_cost_per_token_above_200k_tokens": 0.00024000000000000003, - "input_cost_per_token_flex": 4.5e-05, - "input_cost_per_token_priority": 5.1e-05, - "litellm_provider": "openai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_reasoning_token": 0.00015000000000000001, - "output_cost_per_token": 6.000000000000001e-05, - "output_cost_per_token_above_200k_tokens": 0.00027, - "output_cost_per_token_flex": 7.500000000000001e-05, - "output_cost_per_token_priority": 8.099999999999999e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "gpt-5.4-mini": { - "cache_creation_input_token_cost": 0.00012, - "cache_creation_input_token_cost_above_1hr": 0.00016, - "cache_read_input_token_cost": 4e-06, - "input_cost_per_audio_token": 0.00024, - "input_cost_per_token": 4e-05, - "input_cost_per_token_above_200k_tokens": 0.00032, - "input_cost_per_token_flex": 6e-05, - "input_cost_per_token_priority": 6.8e-05, - "litellm_provider": "openai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "gemini-3.8-flash": { + "cache_read_input_token_cost": 5.2e-08, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 1.04e-06, + "input_cost_per_token": 5.2e-07, + "input_cost_per_token_flex": 2.6e-07, + "input_cost_per_token_priority": 6.5e-07, + "input_cost_per_video_token": 6.2e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00028, - "output_cost_per_reasoning_token": 0.0002, - "output_cost_per_token": 8e-05, - "output_cost_per_token_above_200k_tokens": 0.00036, - "output_cost_per_token_flex": 0.0001, - "output_cost_per_token_priority": 0.000108, + "output_cost_per_audio_token": 6.24e-06, + "output_cost_per_token": 3.12e-06, + "output_cost_per_token_flex": 1.56e-06, + "output_cost_per_token_priority": 3.9e-06, "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 + "search_context_size_medium": 0.035 }, "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "gpt-5.5-pro": { - "cache_read_input_token_cost": 2e-06, - "input_cost_per_token": 2e-05, - "input_cost_per_token_above_200k_tokens": 0.00016, - "input_cost_per_token_flex": 3e-05, - "input_cost_per_token_priority": 3.4e-05, - "litellm_provider": "openai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "output_cost_per_reasoning_token": 0.0001, - "output_cost_per_token": 4e-05, - "output_cost_per_token_above_200k_tokens": 0.00018, - "output_cost_per_token_flex": 5e-05, - "output_cost_per_token_priority": 5.4e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "gpt-5.6": { - "cache_creation_input_token_cost": 3e-05, - "cache_creation_input_token_cost_above_1hr": 4e-05, - "cache_read_input_token_cost": 1e-06, - "input_cost_per_audio_token": 6e-05, - "input_cost_per_token": 1e-05, - "input_cost_per_token_above_200k_tokens": 8e-05, - "input_cost_per_token_flex": 1.5e-05, - "input_cost_per_token_priority": 1.7e-05, - "litellm_provider": "openai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_audio_token": 7e-05, - "output_cost_per_reasoning_token": 5e-05, - "output_cost_per_token": 2e-05, - "output_cost_per_token_above_200k_tokens": 9e-05, - "output_cost_per_token_flex": 2.5e-05, - "output_cost_per_token_priority": 2.7e-05, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "input_cost_per_token": 0.00019, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "output_cost_per_token": 0.00038, - "supports_function_calling": true + "web_search_billing_unit": "per_prompt" }, "together_ai/moonshotai/Kimi-K3": { - "cache_creation_input_token_cost": 0.00030000000000000003, - "cache_creation_input_token_cost_above_1hr": 0.0004, - "cache_read_input_token_cost": 9.999999999999999e-06, - "input_cost_per_audio_token": 0.0006000000000000001, - "input_cost_per_token": 0.0001, - "input_cost_per_token_above_200k_tokens": 0.0008, - "input_cost_per_token_flex": 0.00015000000000000001, - "input_cost_per_token_priority": 0.00017, + "input_cost_per_token": 1.15e-06, "litellm_provider": "together_ai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.0006999999999999999, - "output_cost_per_reasoning_token": 0.0005, - "output_cost_per_token": 0.0002, - "output_cost_per_token_above_200k_tokens": 0.0009000000000000001, - "output_cost_per_token_flex": 0.00025, - "output_cost_per_token_priority": 0.00027, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "output_cost_per_token": 3.45e-06, + "supports_function_calling": true }, "together_ai/zai-org/GLM-5.3": { - "cache_creation_input_token_cost": 0.00033, - "cache_creation_input_token_cost_above_1hr": 0.00044, - "cache_read_input_token_cost": 1.1e-05, - "input_cost_per_audio_token": 0.00066, - "input_cost_per_token": 0.00011, - "input_cost_per_token_above_200k_tokens": 0.00088, - "input_cost_per_token_flex": 0.000165, - "input_cost_per_token_priority": 0.000187, + "input_cost_per_token": 5.5e-07, "litellm_provider": "together_ai", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_audio_token": 0.00077, - "output_cost_per_reasoning_token": 0.00055, - "output_cost_per_token": 0.00022, - "output_cost_per_token_above_200k_tokens": 0.00099, - "output_cost_per_token_flex": 0.000275, - "output_cost_per_token_priority": 0.000297, - "search_context_cost_per_query": { - "search_context_size_high": 0.03, - "search_context_size_low": 0.01, - "search_context_size_medium": 0.02 - }, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true, - "supports_web_search": true + "output_cost_per_token": 2.2e-06, + "supports_function_calling": true }, - "us.anthropic.claude-opus-5-v1:0": { - "cache_creation_input_token_cost": 0.00054, - "cache_creation_input_token_cost_above_1hr": 0.00072, - "cache_read_input_token_cost": 1.8e-05, - "input_cost_per_token": 0.00018, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 2000000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, "mode": "chat", - "output_cost_per_token": 0.00036, - "supports_function_calling": true, - "supports_prompt_caching": true, - "supports_reasoning": true + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "cache_read_input_token_cost": 9e-08, + "input_cost_per_token": 9e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true } } diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 98fcc1b1f04..b0ff6fdcd86 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -186,6 +186,18 @@ class ChatMetadata(BaseModel): class ImageUrl(BaseModel): url: str + detail: str | None = None + + +class InputAudio(BaseModel): + data: str + format: str + + +class FileObject(BaseModel): + file_data: str | None = None + file_id: str | None = None + format: str | None = None class TextContentPart(BaseModel): @@ -199,7 +211,17 @@ class ImageContentPart(BaseModel): image_url: ImageUrl -ContentPart = TextContentPart | ImageContentPart +class InputAudioContentPart(BaseModel): + type: str = "input_audio" + input_audio: InputAudio + + +class FileContentPart(BaseModel): + type: str = "file" + file: FileObject + + +ContentPart = TextContentPart | ImageContentPart | InputAudioContentPart | FileContentPart class ChatMessage(BaseModel): @@ -284,6 +306,37 @@ class ChatToolResultTurn(BaseModel): type ChatTurn = ChatMessage | ChatAssistantTurn | ChatToolResultTurn +class HostedWebSearchTool(BaseModel): + """A provider-hosted web-search tool sent inside an OpenAI tools list + (Anthropic's ``web_search_20250305`` shape).""" + + type: str + name: str + max_uses: int | None = None + + +class GoogleSearchTool(BaseModel): + googleSearch: dict[str, object] = {} + + +class GoogleMapsTool(BaseModel): + googleMaps: dict[str, object] = {} + + +class FileSearchTool(BaseModel): + type: Literal["file_search"] = "file_search" + vector_store_ids: list[str] + + +class WebSearchOptions(BaseModel): + search_context_size: Literal["low", "medium", "high"] | None = None + + +class ChatAudio(BaseModel): + voice: str + format: str + + class ChatStreamOptions(BaseModel): include_usage: bool @@ -302,10 +355,16 @@ class ChatBody(BaseModel): thinking: ThinkingParam | None = None service_tier: str | None = None prompt_cache_key: str | None = None - tools: Sequence[ChatTool | McpChatTool] | None = None + tools: Sequence[ + ChatTool | McpChatTool | HostedWebSearchTool | GoogleSearchTool | GoogleMapsTool | FileSearchTool + ] | None = None tool_choice: str | None = None + modalities: list[str] | None = None + audio: ChatAudio | None = None + web_search_options: WebSearchOptions | None = None guardrails: list[str] | None = None response_format: dict[str, object] | None = None + allowed_openai_params: list[str] | None = None chat_template_kwargs: dict[str, bool] | None = None cache: dict[str, bool] | None = {"no-cache": True} From 5f3a86aee5d7be88c1ef90b211297cc1fd7280f4 Mon Sep 17 00:00:00 2001 From: kerry Date: Fri, 18 Sep 2026 13:27:14 +0000 Subject: [PATCH 20/30] test(e2e): use TypeAlias over 3.12 type statements in e2e models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/models.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/models.py b/tests/e2e/models.py index b0ff6fdcd86..b99d2304289 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -8,7 +8,7 @@ from __future__ import annotations from collections.abc import Sequence from datetime import datetime -from typing import Final, Literal +from typing import Final, Literal, TypeAlias from e2e_http import PartialBody from pydantic import ( @@ -203,7 +203,7 @@ class FileObject(BaseModel): class TextContentPart(BaseModel): type: str = "text" text: str - cache_control: "CacheControl | None" = None + cache_control: CacheControl | None = None class ImageContentPart(BaseModel): @@ -303,7 +303,7 @@ class ChatToolResultTurn(BaseModel): content: str -type ChatTurn = ChatMessage | ChatAssistantTurn | ChatToolResultTurn +ChatTurn: TypeAlias = ChatMessage | ChatAssistantTurn | ChatToolResultTurn class HostedWebSearchTool(BaseModel): @@ -531,7 +531,7 @@ class AnthropicCustomTool(BaseModel): input_schema: ToolInputSchema -type AnthropicTool = AnthropicToolSearchTool | AnthropicWebSearchTool | AnthropicCustomTool +AnthropicTool: TypeAlias = AnthropicToolSearchTool | AnthropicWebSearchTool | AnthropicCustomTool class AnthropicContentBlock(BaseModel): @@ -569,7 +569,7 @@ class AnthropicToolResultTurn(BaseModel): content: list[AnthropicToolResultBlock] -type AnthropicMessage = ChatMessage | AnthropicAssistantTurn | AnthropicToolResultTurn +AnthropicMessage: TypeAlias = ChatMessage | AnthropicAssistantTurn | AnthropicToolResultTurn class AnthropicToolChoice(BaseModel): From 69f9106759aa52375fc167de7059efcb10038400 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 00:16:12 +0000 Subject: [PATCH 21/30] test(integration): move scripted-provider cost suite into cost shard Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- .circleci/scripts/run_integration.sh | 33 +- .../scripts/wait_integration_services.py | 5 + tests/e2e/CLAUDE.md | 3 +- tests/e2e/conftest.py | 8 - tests/e2e/cost_calculation/conftest.py | 185 --- tests/e2e/cost_calculation/scripted_client.py | 64 - .../test_token_pricing_e2e.py | 285 ----- .../coverage_registry/quota_management.yaml | 2 - tests/e2e/e2e_config.py | 16 - .../gateway/cost_calculation_ci_config.yml | 7 - tests/e2e/models.py | 74 +- tests/e2e/pytest.ini | 1 - tests/integration/README.md | 2 + tests/integration/_support/manifest.py | 1 + tests/integration/_support/scripted_client.py | 57 + .../_support}/scripted_provider.py | 21 +- tests/integration/contracts.json | 1092 +++++++++++++++++ .../cost_calculation/cases.json | 0 .../integration/cost_calculation/conftest.py | 147 +++ .../cost_calculation}/cost_map.json | 0 .../cost_calculation/cost_matrix.py | 10 +- .../cost_calculation/test_token_pricing.py | 223 ++++ 23 files changed, 1586 insertions(+), 652 deletions(-) delete mode 100644 tests/e2e/cost_calculation/conftest.py delete mode 100644 tests/e2e/cost_calculation/scripted_client.py delete mode 100644 tests/e2e/cost_calculation/test_token_pricing_e2e.py delete mode 100644 tests/e2e/gateway/cost_calculation_ci_config.yml create mode 100644 tests/integration/_support/scripted_client.py rename tests/{e2e/cost_calculation => integration/_support}/scripted_provider.py (98%) rename tests/{e2e => integration}/cost_calculation/cases.json (100%) create mode 100644 tests/integration/cost_calculation/conftest.py rename tests/{e2e => integration/cost_calculation}/cost_map.json (100%) rename tests/{e2e => integration}/cost_calculation/cost_matrix.py (98%) create mode 100644 tests/integration/cost_calculation/test_token_pricing.py diff --git a/.circleci/config.yml b/.circleci/config.yml index df17a9e4402..6e089436920 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3009,7 +3009,7 @@ workflows: name: integration-<< matrix.suite >> matrix: parameters: - suite: [management, accounting, database, providers, extensions, sdk, browser] + suite: [management, accounting, database, providers, extensions, sdk, cost, browser] filters: branches: only: diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 6fab6dd57db..17850bef4da 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -11,6 +11,7 @@ results="test-results/integration-${suite}" mkdir -p "$results" integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')" upstream_pid="" +scripted_provider_pid="" proxy_pid="" peer_pid="" launched_pid="" @@ -22,9 +23,9 @@ cleanup() { original_status=$? trap - EXIT INT TERM sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \ - "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \ + "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" "$scripted_provider_pid" \ > "$results/process-cleanup.txt" 2>&1 || original_status=1 - for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do + for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid" "$scripted_provider_pid"; do if [ -n "$owned_pid" ]; then kill -- "-$owned_pid" 2>/dev/null || true for _ in {1..50}; do @@ -69,6 +70,7 @@ export STORE_MODEL_IN_DB=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 export INTEGRATION_PROXY_URL=http://127.0.0.1:4000 export INTEGRATION_PEER_URL="" export INTEGRATION_UPSTREAM_URL=http://127.0.0.1:8190 +export INTEGRATION_SCRIPTED_PROVIDER_URL="" export INTEGRATION_MASTER_KEY="$LITELLM_MASTER_KEY" export LITELLM_UI_PATH="$PWD/litellm/proxy/_experimental/out" if [ "$suite" = browser ]; then @@ -108,13 +110,37 @@ awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/e setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \ .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 & upstream_pid=$! +if [ "$suite" = cost ]; then + export INTEGRATION_SCRIPTED_PROVIDER_URL=http://127.0.0.1:8191 + setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \ + .venv/bin/python -m integration._support.scripted_provider --port 8191 \ + > "$results/scripted-provider.log" 2>&1 & + scripted_provider_pid=$! + for _ in {1..90}; do + if curl --noproxy '*' -fsS "$INTEGRATION_SCRIPTED_PROVIDER_URL/health" >/dev/null 2>&1; then + break + fi + sleep 1 + done + curl --noproxy '*' -fsS "$INTEGRATION_SCRIPTED_PROVIDER_URL/health" >/dev/null +fi start_proxy() { local port="$1" local log_name="$2" + local -a cost_map_env + if [ "$suite" = cost ]; then + cost_map_env=( + "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_SCRIPTED_PROVIDER_URL/_cost_map" + "MODEL_COST_MAP_MIN_MODEL_COUNT=1" + "MODEL_COST_MAP_MAX_SHRINK_RATIO=0" + ) + else + cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True") + fi setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \ DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \ - LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True STORE_MODEL_IN_DB=True \ + LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \ AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \ .venv/bin/python -m integration._support.proxy --config tests/integration/proxy_config.yaml \ --host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \ @@ -160,6 +186,7 @@ timeout --signal=TERM --kill-after=20s 11m env -i PATH="$PATH" HOME="$HOME" PYTH DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \ INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \ + INTEGRATION_SCRIPTED_PROVIDER_URL="$INTEGRATION_SCRIPTED_PROVIDER_URL" \ INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ diff --git a/.circleci/scripts/wait_integration_services.py b/.circleci/scripts/wait_integration_services.py index 486e37cba00..462874e8aa6 100644 --- a/.circleci/scripts/wait_integration_services.py +++ b/.circleci/scripts/wait_integration_services.py @@ -9,6 +9,7 @@ from redis import Redis def main() -> None: primary: Final = os.environ["INTEGRATION_PROXY_URL"] peer: Final = os.environ.get("INTEGRATION_PEER_URL") + scripted_provider: Final = os.environ.get("INTEGRATION_SCRIPTED_PROVIDER_URL") or None proxies: Final = (primary, peer) if peer else (primary,) deadline: Final = time.monotonic() + 90 headers: Final = {"Authorization": f"Bearer {os.environ['INTEGRATION_MASTER_KEY']}"} @@ -19,6 +20,10 @@ def main() -> None: try: ready: Final = ( client.get(f"{os.environ['INTEGRATION_UPSTREAM_URL']}/health").status_code == 200 + and ( + scripted_provider is None + or client.get(f"{scripted_provider}/health").status_code == 200 + ) and all(client.get(f"{url}/health/readiness").status_code == 200 for url in proxies) ) if ready: diff --git a/tests/e2e/CLAUDE.md b/tests/e2e/CLAUDE.md index 54c143c11d9..0541ce25d4b 100644 --- a/tests/e2e/CLAUDE.md +++ b/tests/e2e/CLAUDE.md @@ -21,7 +21,6 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `gateway/` - proxy configuration only (`litellm-config.yml`); no tests -- `cost_calculation/` - cost accounting against a dedicated proxy whose whole model cost map is the test-owned `tests/e2e/cost_map.json` (loaded via `LITELLM_MODEL_COST_MAP_URL`), with provider calls answered by the scripted-provider sidecar in `scripted_provider.py`; every cost-map entry is a deployment and the cases plus asserted goldens are data in `cases.json` (each `pricing` case owns (model, cost key) pairs via `owns`/`fallback_for` so every rate key present on each map entry has exactly one owning case, and each carries a literal `expected` cell per map key; `transport` cases list `models` and exercise token counting only; `cost_matrix.matrix_data_errors()` runs at collection time so a key absent from the cost map, an unowned or double-owned (model, rate key) pair, an `owns` key absent on all of the case's models, or a `fallback_for` key present on a case model fails the suite's collection loudly), deselected unless `E2E_COST_MAP_STACK` is set, driven by the Buildkite `e2e-cost-calculation` step in project-releaser, which runs a proxy booted from `gateway/cost_calculation_ci_config.yml` with `MODEL_COST_MAP_MIN_MODEL_COUNT=1` and `MODEL_COST_MAP_MAX_SHRINK_RATIO=0` (the 21-entry test map trips the fetched-cost-map integrity check at the defaults), Postgres and the scripted provider co-located with pytest in one pod and sets the opt-in - `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke - `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json` @@ -222,7 +221,7 @@ other... ``` ## Hard Rules -- no unit tests of any kind under `tests/e2e`. a product feature is proven end to end against a live proxy, never with a unit test, and the harness itself is not unit-tested here either. no monkeypatching or mock tests; the one carve-out is a scripted upstream served through a real HTTP sidecar (the cost_calculation suite's scripted provider), allowed because provider-response-shape coverage needs a controlled usage payload and every hop from the proxy's upstream call to the spend row still executes for real. if a contributor asks you to write an end to end test, do NOT stage a unit test with it; if you find a product gap, call it out in the PR description +- no unit tests of any kind under `tests/e2e`. a product feature is proven end to end against a live proxy, never with a unit test, and the harness itself is not unit-tested here either. no monkeypatching or mock tests. if a contributor asks you to write an end to end test, do NOT stage a unit test with it; if you find a product gap, call it out in the PR description - use model management endpoints to create new models for a test. this could be in a conftest / inline for each test. ask the user what they want. diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index b7f8d8611a4..e83827fac74 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -25,7 +25,6 @@ import requests from e2e_config import ( CLI_DETERMINISM_OPT_IN_ENV, CONTROL_PLANE_BASE_URL, - COST_MAP_OPT_IN_ENV, FIXTURE_DIR, FIXTURE_MODE_RAW, MANAGED_FILES_OPT_IN_ENV, @@ -56,7 +55,6 @@ OPT_IN_MARKERS: Final = MappingProxyType( "managed_files": MANAGED_FILES_OPT_IN_ENV, "prompt_caching_stack": PROMPT_CACHING_OPT_IN_ENV, "redis_chaos": REDIS_CHAOS_OPT_IN_ENV, - "cost_map_stack": COST_MAP_OPT_IN_ENV, "cli_determinism": CLI_DETERMINISM_OPT_IN_ENV, } ) @@ -134,12 +132,6 @@ def pytest_configure(config: pytest.Config) -> None: "redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from " "gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set", ) - config.addinivalue_line( - "markers", - "cost_map_stack: needs a proxy whose whole cost map is tests/e2e/cost_map.json " - "(LITELLM_MODEL_COST_MAP_URL) plus a scripted-provider sidecar; deselected unless " - "E2E_COST_MAP_STACK is set", - ) def pytest_sessionstart(session: pytest.Session) -> None: diff --git a/tests/e2e/cost_calculation/conftest.py b/tests/e2e/cost_calculation/conftest.py deleted file mode 100644 index e735de40027..00000000000 --- a/tests/e2e/cost_calculation/conftest.py +++ /dev/null @@ -1,185 +0,0 @@ -"""Cost-calculation suite fixtures. - -Runs against a dedicated proxy whose whole model cost map is the test-owned -``tests/e2e/cost_map.json`` (LITELLM_MODEL_COST_MAP_URL); every map entry is a -deployment under test, and the request shapes plus asserted goldens live in -``cases.json``. Provider calls are answered by the -scripted-provider sidecar (``scripted_provider.py``), registered per scenario -over its control API. - -The proxy must also run with ``MODEL_COST_MAP_MIN_MODEL_COUNT=1`` and -``MODEL_COST_MAP_MAX_SHRINK_RATIO=0``: the 21-entry test map trips the -fetched-cost-map integrity check (too few models, large shrink versus the -bundled map) at those env vars' defaults. - -Deselected unless E2E_COST_MAP_STACK is set (marker `cost_map_stack`). -""" - -from __future__ import annotations - -import functools -import importlib.util -import json -import sys -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from pathlib import Path -from types import ModuleType -from typing import Final, Protocol, cast - -import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa - -from cost_matrix import Case, FrontierModel -from e2e_config import COST_MAP_PROXY_URL, SCRIPTED_PROVIDER_PROXY_BASE -from lifecycle import ResourceManager -from models import LiteLLMParamsBody, ModelInfoBody, ModelNewBody -from proxy_client import ProxyClient, build_proxy_client -from scripted_client import ScenarioHandle, delete_scenario, register_scenario -from scripted_provider import Scenario - - -def _load_cost_rows() -> ModuleType: - """Load quota_management/spend_tracking/cost_rows.py by path (the e2e tree - has no package layout), the same trick the mcp suite uses for - logging/datadog_reader.py.""" - path: Final = ( - Path(__file__).resolve().parent.parent - / "quota_management" - / "spend_tracking" - / "cost_rows.py" - ) - name: Final = "e2e_spend_tracking_cost_rows" - spec: Final = importlib.util.spec_from_file_location(name, path) - assert spec is not None and spec.loader is not None - module: Final = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - return module - - -class SpendCostBreakdown(Protocol): - input_cost: float | None - output_cost: float | None - cache_read_cost: float | None - cache_creation_cost: float | None - reasoning_cost: float | None - tool_usage_cost: float | None - total_cost: float | None - service_tier: str | None - - def model_dump(self) -> Mapping[str, object]: ... - - -class SpendRowMetadata(Protocol): - cost_breakdown: SpendCostBreakdown | None - - -class SpendCostRow(Protocol): - """The slice of spend_tracking.cost_rows.CostRow this suite reads.""" - - spend: float | None - prompt_tokens: int | None - completion_tokens: int | None - metadata: SpendRowMetadata | None - - @property - def breakdown(self) -> SpendCostBreakdown: ... - - -class CostRowsModule(Protocol): - """cost_rows.py loaded by path has no importable name for basedpyright, so - its surface is declared here and reached through a single cast.""" - - approx_equal: Callable[[float, float], bool] - assert_total_is_sum_of_components: Callable[[SpendCostRow], None] - poll_cost_row_where: Callable[ - [ProxyClient, str, Callable[[SpendCostRow], bool]], SpendCostRow | None - ] - - -cost_rows: Final[CostRowsModule] = cast( # cast-ok: cost_rows.py is loaded by path, so basedpyright has no importable name for it; its surface is declared in CostRowsModule - CostRowsModule, _load_cost_rows() -) - - -@dataclass(frozen=True, slots=True) -class CostCalcClient: - """The suite's client: a ProxyClient pointed at the cost-map proxy pod.""" - - proxy: ProxyClient - - -@pytest.fixture(scope="session") -def client() -> CostCalcClient: - proxy: Final = build_proxy_client( - base_url=COST_MAP_PROXY_URL, - control_plane_base_url=COST_MAP_PROXY_URL, - replica_urls=(COST_MAP_PROXY_URL,), - ) - return CostCalcClient(proxy=proxy) - - -@functools.cache -def _vertex_private_key_pem() -> str: - return rsa.generate_private_key(public_exponent=65537, key_size=2048).private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ).decode() - - -def _vertex_service_account_json() -> str: - """A service-account credential JSON whose token_uri is the sidecar's - /_oauth/token route: the proxy's google-auth refresh then gets a scripted - access token without touching Google.""" - return json.dumps( - { - "type": "service_account", - "project_id": "cc-scripted-project", - "private_key_id": "scripted", - "private_key": _vertex_private_key_pem(), - "client_email": "scripted@cc-scripted-project.iam.gserviceaccount.com", - "client_id": "0", - "auth_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/authorize", - "token_uri": f"{SCRIPTED_PROVIDER_PROXY_BASE}/_oauth/token", - } - ) - - -def register_scenario_deployment( - client: CostCalcClient, - resources: ResourceManager, - model: FrontierModel, - case: Case, - marker: str, -) -> tuple[str, ScenarioHandle]: - """Register the case's scenario on the sidecar plus a deployment pointed at - it; both are torn down by ``resources``. Returns the callable model_name.""" - scenario: Final[Scenario] = case.scenario( - scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" - ) - handle: Final = register_scenario(scenario) - resources.defer(lambda: delete_scenario(handle)) - model_name: Final = f"{model.model_name}-{marker}" - params: Final = { - "model": model.litellm_model, - "api_key": model.api_key, - "api_base": handle.api_base(), - **model.litellm_params, - **( - {"vertex_credentials": _vertex_service_account_json()} - if model.wire == "vertex_generate" - else {} - ), - } - model_id: Final = client.proxy.register_model( - ModelNewBody( - model_name=model_name, - litellm_params=LiteLLMParamsBody.model_validate(params), - model_info=ModelInfoBody(base_model=model.base_model), - ) - ) - resources.defer(lambda: client.proxy.delete_model(model_id)) - return model_name, handle diff --git a/tests/e2e/cost_calculation/scripted_client.py b/tests/e2e/cost_calculation/scripted_client.py deleted file mode 100644 index 9dbf9c98986..00000000000 --- a/tests/e2e/cost_calculation/scripted_client.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Client side of the scripted-provider sidecar: register scenarios over its -control API through the shared transport helpers and get back a handle whose -``api_base`` is what a /model/new deployment should register for the proxy to -reach the scripted wire.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Final - -from e2e_config import SCRIPTED_PROVIDER_CONTROL_URL, SCRIPTED_PROVIDER_PROXY_BASE -from e2e_http import URL, NoBody, unwrap, post -from e2e_http import delete as http_delete -from scripted_provider import ( - WIRE_MOUNTS, - Scenario, - ScenarioDeleted, - ScenarioRegistered, - Wire, -) - - -@dataclass(frozen=True, slots=True) -class ScenarioHandle: - scenario_id: str - wire: Wire - proxy_base: str - - def api_base(self) -> str: - return f"{self.proxy_base}/{self.scenario_id}/{self._mount()}" - - def _mount(self) -> str: - return WIRE_MOUNTS[self.wire] - - -def register_scenario(scenario: Scenario) -> ScenarioHandle: - """POST the scenario to the sidecar's control API and return its handle.""" - result: Final = unwrap( - post( - URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios"), - headers=NoBody(), - json=scenario, - response_type=ScenarioRegistered, - ) - ) - return ScenarioHandle( - scenario_id=result.scenario_id, - wire=scenario.wire, - proxy_base=SCRIPTED_PROVIDER_PROXY_BASE, - ) - - -def delete_scenario(handle: ScenarioHandle) -> None: - unwrap( - http_delete( - URL(f"{SCRIPTED_PROVIDER_CONTROL_URL}/_scenarios/{handle.scenario_id}"), - headers=NoBody(), - json=NoBody(), - response_type=ScenarioDeleted, - ) - ) - - -CONTROL_URL: Final = SCRIPTED_PROVIDER_CONTROL_URL diff --git a/tests/e2e/cost_calculation/test_token_pricing_e2e.py b/tests/e2e/cost_calculation/test_token_pricing_e2e.py deleted file mode 100644 index 004cb4d839e..00000000000 --- a/tests/e2e/cost_calculation/test_token_pricing_e2e.py +++ /dev/null @@ -1,285 +0,0 @@ -"""Token-pricing e2e: every (map entry, case) cell derived from cost_map.json x -cases.json runs a scripted-usage call through a deployment registered on the -cost-map proxy, and the spend row plus response-cost header must equal the -reviewed golden in the case's ``expected`` cell verbatim -- no rate arithmetic -lives here. - -Nothing here touches a real provider or the bundled cost map: the proxy's -upstream is the scripted-provider sidecar and its entire cost map is -tests/e2e/cost_map.json. -""" - -from __future__ import annotations - -import pytest -from typing import Final - -from conftest import CostCalcClient, cost_rows, register_scenario_deployment -from cost_matrix import ( - AUDIO_INPUT_DATA_URL, - FRONTIER_MODELS, - IMAGE_INPUT_DATA_URL, - SERVICE_TIER_REQUEST_WIRES, - VIDEO_INPUT_DATA_URL, - Case, - FrontierModel, - cases_for, - matrix_data_errors, - recount_cost, -) -from e2e_config import unique_marker -from lifecycle import ResourceManager -from models import ( - CacheControl, - ChatAudio, - ChatBody, - ChatMessage, - ChatStreamOptions, - ChatTool, - ChatToolFunction, - FileContentPart, - FileObject, - FileSearchTool, - GoogleMapsTool, - GoogleSearchTool, - HostedWebSearchTool, - ImageContentPart, - ImageUrl, - InputAudio, - InputAudioContentPart, - TextContentPart, - WebSearchOptions, -) -from scripted_provider import ScriptedUsage, Wire - -pytestmark: Final = [pytest.mark.e2e, pytest.mark.cost_map_stack] # mutable-ok: pytest only accepts a list for pytestmark - -if _data_errors := matrix_data_errors(): - raise ValueError("\n".join(_data_errors)) - -_MATRIX: Final[tuple[tuple[FrontierModel, Case], ...]] = tuple( - (model, case) for model in FRONTIER_MODELS for case in cases_for(model) -) - - -def _case_id(param: tuple[FrontierModel, Case]) -> str: - model, case = param - return f"{model.map_key.replace('/', '-')}-{case.name}" - - -_CACHE_WIRES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) -_WEB_SEARCH_OPTION_WIRES: Final = frozenset({"openai_chat", "azure_chat", "openai_responses"}) - - -def _cache_control(usage: ScriptedUsage, wire: Wire) -> CacheControl | None: - if wire not in _CACHE_WIRES: - return None - if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens): - return None - return CacheControl(type="ephemeral", ttl="1h" if usage.cache_write_1h_tokens else None) - - -def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) -> ChatBody: - usage: Final = case.usage_for(model.map_key) - user_parts: Final = ( - TextContentPart( - text=f"{marker} summarize the attached material in one line and name the city weather", - ), - *( - (ImageContentPart(image_url=ImageUrl(url=IMAGE_INPUT_DATA_URL, detail="high")),) - if case.image_input - else () - ), - *( - ( - InputAudioContentPart( - input_audio=InputAudio(data=AUDIO_INPUT_DATA_URL.split(",", 1)[1], format="wav") - ), - ) - if case.audio_input - else () - ), - *( - (FileContentPart(file=FileObject(file_data=VIDEO_INPUT_DATA_URL, format="mp4")),) - if case.video_input - else () - ), - ) - tools: Final = ( - *( - ( - ChatTool( - function=ChatToolFunction( - name="get_weather", - description="Get the current weather and a short forecast for a city.", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"}, - "days": {"type": "integer", "description": "Forecast horizon in days"}, - "units": {"type": "string", "enum": ["metric", "imperial"]}, - }, - "required": ["city"], - }, - ) - ), - ) - if case.tool_call - else () - ), - *( - (HostedWebSearchTool(type="web_search_20250305", name="web_search", max_uses=5),) - if case.web_search is not None and model.wire == "anthropic_messages" - else () - ), - *( - (GoogleSearchTool(),) - if case.web_search is not None and model.wire in ("gemini_generate", "vertex_generate") - else () - ), - *((GoogleMapsTool(),) if case.google_maps else ()), - *((FileSearchTool(vector_store_ids=["vs_cost_calc_fixture"]),) if case.file_search else ()), - ) - return ChatBody( - model=model_name, - messages=( - ChatMessage( - role="system", - content=[ - TextContentPart( - text=( - "You are a deterministic pricing-harness assistant. " - "Keep answers to a single short line." - ), - cache_control=_cache_control(usage, model.wire), - ) - ], - ), - ChatMessage(role="user", content=list(user_parts)), - ), - stream=case.stream, - stream_options=ChatStreamOptions(include_usage=True) if case.stream else None, - service_tier=( - case.service_tier - if case.service_tier is not None and model.wire in SERVICE_TIER_REQUEST_WIRES - else None - ), - reasoning_effort="medium" if case.reasoning else None, - modalities=( - ["text"] if case.audio_input else (["text", "audio"] if case.audio_output else None) - ), - audio=( - ChatAudio(voice="alloy", format="pcm16") if case.audio_output else None - ), - web_search_options=( - WebSearchOptions(search_context_size=case.web_search) - if case.web_search is not None and model.wire in _WEB_SEARCH_OPTION_WIRES - else None - ), - tools=tools or None, - tool_choice="auto" if case.tool_call and model.wire != "bedrock_converse" else None, - # The test-owned cost map carries no supports_* flags, so litellm's - # optional-params gate rejects the realistic request fields; allowlist - # exactly the ones this case sends. - allowed_openai_params=[ - name - for name, sent in ( - ("tool_choice", case.tool_call and model.wire != "bedrock_converse"), - ("modalities", case.audio_input or case.audio_output), - ("audio", case.audio_output), - ("web_search_options", case.web_search is not None), - ("reasoning_effort", case.reasoning), - ) - if sent - ], - ) - - -class TestTokenPricing: - @pytest.mark.parametrize("model_case", _MATRIX, ids=_case_id) - @pytest.mark.covers("quota_management.spend_tracking.cost_matrix.logs_cost") - def test_scripted_usage_bills_at_map_rates( - self, - client: CostCalcClient, - resources: ResourceManager, - scoped_key: str, - model_case: tuple[FrontierModel, Case], - ) -> None: - model, case = model_case - marker: Final = unique_marker() - model_name, _handle = register_scenario_deployment(client, resources, model, case, marker) - response: Final = client.proxy.transport.send( - "/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=_chat_body(model, case, model_name, marker), - stream=case.stream, - ) - assert response.ok, ( - f"{model.map_key}/{case.name}: proxy returned {response.status_code}: {response.body[:400]}" - ) - assert response.stream_error is None, f"stream carried an error event: {response.stream_error}" - - row: Final = cost_rows.poll_cost_row_where( - client.proxy, - scoped_key, - lambda r: r.metadata is not None and r.metadata.cost_breakdown is not None, - ) - assert row is not None, f"no spend row with a cost breakdown landed for {model.map_key}/{case.name}" - - if not case.exact_spend: - # stream_usage=absent: the provider reported no usage, so the row's - # token counts are the proxy's own recount; assert the recount - # billed both directions at the case's rates. - assert row.prompt_tokens is not None and row.prompt_tokens > 0, ( - f"no-usage stream counted no input tokens: {row}" - ) - assert row.completion_tokens is not None and row.completion_tokens > 0, ( - f"no-usage stream counted no output tokens: {row}" - ) - if case.image_input: - assert row.prompt_tokens < 4000, ( - f"image data URL looks tokenized as text: prompt_tokens={row.prompt_tokens}" - ) - assert row.spend is not None and cost_rows.approx_equal( - row.spend, - recount_cost(model, case, row.prompt_tokens, row.completion_tokens), - ), f"no-usage stream spend {row.spend} != recount at map rates: {row}" - cost_rows.assert_total_is_sum_of_components(row) - return - - golden: Final = case.expected_for(model) - - if not case.stream: - # Streamed responses commit headers before the bill is computed, so - # the x-litellm-response-cost header is asserted only on non-stream - # calls. - assert response.response_cost is not None and cost_rows.approx_equal( - response.response_cost, golden.spend - ), ( - f"x-litellm-response-cost {response.response_cost} != golden {golden.spend}" - ) - - assert row.spend is not None and cost_rows.approx_equal(row.spend, golden.spend), ( - f"{model.map_key}/{case.name}: spend {row.spend} != golden {golden.spend} " - f"(breakdown {row.breakdown.model_dump()})" - ) - breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and cost_rows.approx_equal( - breakdown.input_cost, golden.input_cost - ), ( - f"{model.map_key}/{case.name}: gross input_cost {breakdown.input_cost} " - f"!= golden {golden.input_cost}; cached/written tokens billed at the input rate" - ) - assert breakdown.output_cost is not None and cost_rows.approx_equal( - breakdown.output_cost, golden.output_cost - ), ( - f"{model.map_key}/{case.name}: output_cost {breakdown.output_cost} " - f"!= golden {golden.output_cost}" - ) - assert row.prompt_tokens == golden.prompt_tokens, ( - f"prompt_tokens {row.prompt_tokens} != {golden.prompt_tokens}" - ) - assert row.completion_tokens == golden.completion_tokens, ( - f"completion_tokens {row.completion_tokens} != {golden.completion_tokens}" - ) - cost_rows.assert_total_is_sum_of_components(row) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 6b40e70125c..ad0914d455b 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -63,5 +63,3 @@ - {id: quota_management.spend_tracking.key_attribution.health_rows_keep_service_account, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [health_rows_keep_service_account], exercised_on: [chat_completions], source: "proxy/health_check.py", rationale: "A /health probe's spend row stays keyed by the literal litellm-internal-health-check service account rather than a hash of it, so health spend never appears as an unattributed key"} - {id: quota_management.spend_tracking.key_attribution.retrieve_batch_cost_joins_retrieving_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [retrieve_batch_cost_joins_retrieving_key], exercised_on: [batches], source: "proxy/batches_endpoints/endpoints.py", rationale: "The retrieve that first sees a batch in a terminal state prices it inline and writes its {provider_batch_id}_batch_cost row against the retrieving key, so the batch each run creates is one OpenAI fails at validation within seconds and the test retrieves it by its raw provider id with the same key until it is failed; a raw id is never owned by the CheckBatchCost poller, and the row must carry that key's token hash and alias"} - {id: quota_management.spend_tracking.key_attribution.poller_batch_cost_joins_creating_key, module: quota_management, tier: P1, behavior: spend_tracking, variant: key_attribution, assertions: [poller_batch_cost_joins_creating_key], exercised_on: [batches], source: "enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py", rationale: "The CheckBatchCost poller bills a completed, positive-cost batch created through a unified id against the key that created it, a different writer from the inline retrieve. No test claims this cell yet: OpenAI's completion window is 24h and both e2e stacks boot a fresh Postgres per build, so a completed batch is out of one run's reach and the managed list never shows an earlier run's batch; the cell stays visible as a gap until a run can hand a completed batch to the poller"} -- {id: quota_management.spend_tracking.cost_matrix.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_matrix, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "A scripted-usage call through the cost-map proxy bills every reported token kind at the deployment's test-map rate (input, output, cache read, 5m/1h cache write, reasoning, audio, above-threshold tiers, flex/priority service tiers, web search, response-model override) and lands on the row's cost_breakdown, streamed or not"} -- {id: quota_management.spend_tracking.scripted_wire.logs_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: scripted_wire, assertions: [logs_cost], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "Each provider wire shape (openai chat, responses, anthropic messages, gemini generateContent, together, fireworks) parses usage into the same spend components: the gross input cost is fresh tokens at the input rate plus each cache/audio component at its own rate, streamed anthropic included"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 370cb9a242f..11c52d1398c 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -143,22 +143,6 @@ WEEKLY_ANOMALY_OPT_IN_ENV = "E2E_WEEKLY_ANOMALY" MANAGED_FILES_OPT_IN_ENV = "E2E_MANAGED_FILES_STACK" PROMPT_CACHING_OPT_IN_ENV = "E2E_PROMPT_CACHING_STACK" REDIS_CHAOS_OPT_IN_ENV = "E2E_REDIS_CHAOS" -# The cost_calculation suite needs a proxy booted with LITELLM_MODEL_COST_MAP_URL -# pointing at tests/e2e/cost_map.json (its whole map is test-owned rates) plus a -# scripted-provider sidecar; deselected unless the opt-in env var is set. -COST_MAP_OPT_IN_ENV = "E2E_COST_MAP_STACK" -# Base URL of the proxy running the test cost map. Defaults to the shared proxy -# so a local run only has to set the opt-in and boot the proxy accordingly. -COST_MAP_PROXY_URL = os.environ.get("E2E_COST_MAP_PROXY_URL", PROXY_BASE_URL).rstrip("/") -# Where the test runner reaches the scripted-provider sidecar's control API. -SCRIPTED_PROVIDER_CONTROL_URL = os.environ.get( - "E2E_SCRIPTED_PROVIDER_CONTROL_URL", "http://127.0.0.1:9100" -).rstrip("/") -# The api_base root deployments register with: how the proxy (possibly in -# another container) reaches the sidecar's provider wire. -SCRIPTED_PROVIDER_PROXY_BASE = os.environ.get( - "E2E_SCRIPTED_PROVIDER_PROXY_BASE", SCRIPTED_PROVIDER_CONTROL_URL -).rstrip("/") CLI_DETERMINISM_OPT_IN_ENV = "E2E_CLI_DETERMINISM" ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6")) ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6")) diff --git a/tests/e2e/gateway/cost_calculation_ci_config.yml b/tests/e2e/gateway/cost_calculation_ci_config.yml deleted file mode 100644 index ac0603fa7c1..00000000000 --- a/tests/e2e/gateway/cost_calculation_ci_config.yml +++ /dev/null @@ -1,7 +0,0 @@ -general_settings: - master_key: os.environ/LITELLM_MASTER_KEY - database_url: os.environ/DATABASE_URL - store_model_in_db: true - proxy_batch_write_at: 5 - -model_list: [] diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 9cc28b38d27..9f49c5974d0 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -8,7 +8,7 @@ from __future__ import annotations from collections.abc import Sequence from datetime import datetime -from typing import Final, Literal, TypeAlias +from typing import Final, Literal from e2e_http import PartialBody from pydantic import ( @@ -187,24 +187,12 @@ class ChatMetadata(BaseModel): class ImageUrl(BaseModel): url: str - detail: str | None = None - - -class InputAudio(BaseModel): - data: str - format: str - - -class FileObject(BaseModel): - file_data: str | None = None - file_id: str | None = None - format: str | None = None class TextContentPart(BaseModel): type: str = "text" text: str - cache_control: CacheControl | None = None + cache_control: "CacheControl | None" = None class ImageContentPart(BaseModel): @@ -212,17 +200,7 @@ class ImageContentPart(BaseModel): image_url: ImageUrl -class InputAudioContentPart(BaseModel): - type: str = "input_audio" - input_audio: InputAudio - - -class FileContentPart(BaseModel): - type: str = "file" - file: FileObject - - -ContentPart = TextContentPart | ImageContentPart | InputAudioContentPart | FileContentPart +ContentPart = TextContentPart | ImageContentPart class ChatMessage(BaseModel): @@ -304,38 +282,7 @@ class ChatToolResultTurn(BaseModel): content: str -ChatTurn: TypeAlias = ChatMessage | ChatAssistantTurn | ChatToolResultTurn - - -class HostedWebSearchTool(BaseModel): - """A provider-hosted web-search tool sent inside an OpenAI tools list - (Anthropic's ``web_search_20250305`` shape).""" - - type: str - name: str - max_uses: int | None = None - - -class GoogleSearchTool(BaseModel): - googleSearch: dict[str, object] = {} - - -class GoogleMapsTool(BaseModel): - googleMaps: dict[str, object] = {} - - -class FileSearchTool(BaseModel): - type: Literal["file_search"] = "file_search" - vector_store_ids: list[str] - - -class WebSearchOptions(BaseModel): - search_context_size: Literal["low", "medium", "high"] | None = None - - -class ChatAudio(BaseModel): - voice: str - format: str +type ChatTurn = ChatMessage | ChatAssistantTurn | ChatToolResultTurn class ChatStreamOptions(BaseModel): @@ -356,16 +303,10 @@ class ChatBody(BaseModel): thinking: ThinkingParam | None = None service_tier: str | None = None prompt_cache_key: str | None = None - tools: Sequence[ - ChatTool | McpChatTool | HostedWebSearchTool | GoogleSearchTool | GoogleMapsTool | FileSearchTool - ] | None = None + tools: Sequence[ChatTool | McpChatTool] | None = None tool_choice: str | None = None - modalities: list[str] | None = None - audio: ChatAudio | None = None - web_search_options: WebSearchOptions | None = None guardrails: list[str] | None = None response_format: dict[str, object] | None = None - allowed_openai_params: list[str] | None = None chat_template_kwargs: dict[str, bool] | None = None cache: dict[str, bool] | None = {"no-cache": True} @@ -532,7 +473,7 @@ class AnthropicCustomTool(BaseModel): input_schema: ToolInputSchema -AnthropicTool: TypeAlias = AnthropicToolSearchTool | AnthropicWebSearchTool | AnthropicCustomTool +type AnthropicTool = AnthropicToolSearchTool | AnthropicWebSearchTool | AnthropicCustomTool class AnthropicContentBlock(BaseModel): @@ -570,7 +511,7 @@ class AnthropicToolResultTurn(BaseModel): content: list[AnthropicToolResultBlock] -AnthropicMessage: TypeAlias = ChatMessage | AnthropicAssistantTurn | AnthropicToolResultTurn +type AnthropicMessage = ChatMessage | AnthropicAssistantTurn | AnthropicToolResultTurn class AnthropicToolChoice(BaseModel): @@ -1061,7 +1002,6 @@ class ModelInfoBody(BaseModel): access_groups: list[str] | None = None team_id: str | None = None allowed_fails_policy: dict[str, int] | None = None - base_model: str | None = None class ModelNewBody(BaseModel): diff --git a/tests/e2e/pytest.ini b/tests/e2e/pytest.ini index f05d25a6004..f9e5995079b 100644 --- a/tests/e2e/pytest.ini +++ b/tests/e2e/pytest.ini @@ -12,4 +12,3 @@ markers = prompt_caching_stack: needs a proxy running with router_settings.optional_pre_call_checks including prompt_caching; deselected unless E2E_PROMPT_CACHING_STACK is set cli_determinism: drives the real claude CLI for several seconds; deselected unless E2E_CLI_DETERMINISM is set redis_chaos: load test that pauses the proxy's Redis outright mid-run; needs a proxy booted from gateway/redis_chaos_ci_config.yml on the same host, and is deselected unless E2E_REDIS_CHAOS is set - cost_map_stack: needs a proxy whose whole cost map is tests/e2e/cost_map.json (LITELLM_MODEL_COST_MAP_URL) plus a scripted-provider sidecar; deselected unless E2E_COST_MAP_STACK is set diff --git a/tests/integration/README.md b/tests/integration/README.md index 5ea34fc9180..0049a640111 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,6 +2,8 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls +The `cost` group runs the scripted-provider cost matrix through a dedicated sidecar. The sidecar serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry + Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions` or `sdk` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index 3c9a5508ad6..0117a0df591 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -20,6 +20,7 @@ OWNED_DIRECTORIES: Final = frozenset( "observability", "compatibility", "sdk", + "cost_calculation", } ) diff --git a/tests/integration/_support/scripted_client.py b/tests/integration/_support/scripted_client.py new file mode 100644 index 00000000000..7818488fae0 --- /dev/null +++ b/tests/integration/_support/scripted_client.py @@ -0,0 +1,57 @@ +"""Client for registering scenarios with the integration scripted provider.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import Final + +import httpx +from integration._support.scripted_provider import ( + WIRE_MOUNTS, + Scenario, + ScenarioDeleted, + ScenarioRegistered, + Wire, +) + +CONTROL_URL: Final = os.environ.get("INTEGRATION_SCRIPTED_PROVIDER_URL", "http://127.0.0.1:8191").rstrip("/") + + +@dataclass(frozen=True, slots=True) +class ScenarioHandle: + scenario_id: str + wire: Wire + control_url: str + + def api_base(self) -> str: + return f"{self.control_url}/{self.scenario_id}/{self._mount()}" + + def _mount(self) -> str: + return WIRE_MOUNTS[self.wire] + + +def register_scenario(scenario: Scenario) -> ScenarioHandle: + response: Final = httpx.post( + f"{CONTROL_URL}/_scenarios", + json=scenario.model_dump(mode="json"), + trust_env=False, + timeout=15, + ) + response.raise_for_status() + result: Final = ScenarioRegistered.model_validate_json(response.content) + return ScenarioHandle( + scenario_id=result.scenario_id, + wire=scenario.wire, + control_url=CONTROL_URL, + ) + + +def delete_scenario(handle: ScenarioHandle) -> None: + response: Final = httpx.delete( + f"{CONTROL_URL}/_scenarios/{handle.scenario_id}", + trust_env=False, + timeout=15, + ) + response.raise_for_status() + ScenarioDeleted.model_validate_json(response.content) diff --git a/tests/e2e/cost_calculation/scripted_provider.py b/tests/integration/_support/scripted_provider.py similarity index 98% rename from tests/e2e/cost_calculation/scripted_provider.py rename to tests/integration/_support/scripted_provider.py index c154dcdae62..d5e0fd7e9cf 100644 --- a/tests/e2e/cost_calculation/scripted_provider.py +++ b/tests/integration/_support/scripted_provider.py @@ -1,6 +1,6 @@ -"""Scripted provider sidecar for the cost-calculation e2e suite. +"""Scripted provider sidecar for the cost-calculation integration suite. -A standalone process (``python -m cost_calculation.scripted_provider``) that +A standalone process (``python -m integration._support.scripted_provider``) that pretends to be an LLM provider for the proxy under test. The suite registers a Scenario over a small control API; the provider wire routes then answer the proxy's upstream calls with the scripted usage figures, in the exact wire shape @@ -32,6 +32,7 @@ final stream chunk carries usage or the provider reports none. from __future__ import annotations +import argparse import json import struct import sys @@ -41,8 +42,9 @@ import zlib from collections.abc import Mapping from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias, cast from urllib.parse import unquote, urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, model_validator @@ -1362,6 +1364,12 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte segments: Final = tuple(segment for segment in path.split("/") if segment) if method == "GET" and segments == ("health",): return RenderedResponse(200, "application/json", _json_bytes(_jobj(("status", "ok")))) + if method == "GET" and segments == ("_cost_map",): + return RenderedResponse( + 200, + "application/json", + (Path(__file__).resolve().parents[1] / "cost_calculation" / "cost_map.json").read_bytes(), + ) if segments and segments[0] == "_oauth": if method == "POST" and segments == ("_oauth", "token"): return RenderedResponse( @@ -1459,7 +1467,7 @@ class _ScriptedHandler(BaseHTTPRequestHandler): -DEFAULT_PORT: Final = 9100 +DEFAULT_PORT: Final = 8191 def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: @@ -1469,5 +1477,6 @@ def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: if __name__ == "__main__": - port_arg: Final = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT - serve(port=port_arg) + parser: Final = argparse.ArgumentParser() + parser.add_argument("--port", type=int, default=8191) + serve(port=cast(int, parser.parse_args().port)) diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 91b1bd86954..932ebad9fe1 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -24,6 +24,9 @@ ], "sdk": [ "sdk" + ], + "cost": [ + "cost_calculation" ] }, "tests": { @@ -213,6 +216,1095 @@ ], "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": [ "other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-anthropic_us_inference]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tiered_cache_write_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-anthropic_fast_mode]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-anthropic_us_inference]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tiered_cache_write_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-anthropic_us_inference]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-image_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-video_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-web_search_per_prompt]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-fallback_reasoning_at_output_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-image_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-video_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-image_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-video_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-web_search_per_prompt]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-google_maps_grounding]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_prompt_blocked]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-file_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-file_search]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_incomplete]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_unvalidated]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-audio_input]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-audio_output]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-reasoning]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-web_search_medium]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-web_search_low]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-web_search_high]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-input_text]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-cache_read]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": [ + "quota_management.spend_tracking.cost_matrix.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" + ], + "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ + "quota_management.spend_tracking.scripted_wire.logs_cost" ] }, "browser": { diff --git a/tests/e2e/cost_calculation/cases.json b/tests/integration/cost_calculation/cases.json similarity index 100% rename from tests/e2e/cost_calculation/cases.json rename to tests/integration/cost_calculation/cases.json diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py new file mode 100644 index 00000000000..bc08aa554f5 --- /dev/null +++ b/tests/integration/cost_calculation/conftest.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import functools +import json +import os +from collections.abc import Mapping +from hashlib import sha256 +from typing import Final + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from pydantic import BaseModel, ConfigDict + +from integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.scripted_client import delete_scenario, register_scenario +from integration.cost_calculation.cost_matrix import Case, FrontierModel + + +class CostBreakdown(BaseModel): + model_config = ConfigDict(extra="ignore") + + input_cost: float | None = None + output_cost: float | None = None + cache_read_cost: float | None = None + cache_creation_cost: float | None = None + reasoning_cost: float | None = None + tool_usage_cost: float | None = None + total_cost: float | None = None + service_tier: str | None = None + + +class CostMetadata(BaseModel): + model_config = ConfigDict(extra="ignore") + + cost_breakdown: CostBreakdown | None = None + + +class CostRow(BaseModel): + model_config = ConfigDict(extra="ignore") + + spend: float | None = None + prompt_tokens: int | None = None + completion_tokens: int | None = None + metadata: CostMetadata | None = None + + @property + def breakdown(self) -> CostBreakdown: + assert self.metadata is not None and self.metadata.cost_breakdown is not None + return self.metadata.cost_breakdown + + +def approx_equal(actual: float, expected: float) -> bool: + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +def assert_total_is_sum_of_components(row: CostRow) -> None: + breakdown: Final = row.breakdown + total: Final = sum( + cost or 0.0 + for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) + ) + assert breakdown.total_cost is not None and approx_equal(breakdown.total_cost, total) + assert row.spend is not None and approx_equal(row.spend, breakdown.total_cost) + + +def _row(value: Mapping[str, object]) -> CostRow | None: + metadata_value: Final = value.get("metadata") + metadata: Final = json.loads(metadata_value) if isinstance(metadata_value, str) else metadata_value + parsed: Final = CostRow.model_validate({**value, "metadata": metadata}) + return parsed if parsed.metadata and parsed.metadata.cost_breakdown else None + + +def poll_cost_row(key: str) -> CostRow: + digest: Final = sha256(key.encode()).hexdigest() + + def read() -> CostRow | None: + rows: Final = read_rows( + 'SELECT spend, metadata, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE api_key=%s', + (digest,), + ) + return next((parsed for row in rows if (parsed := _row(row)) is not None), None) + + result: Final = eventually(read, lambda row: row is not None, seconds=60) + assert result is not None + return result + + +@functools.cache +def _vertex_private_key_pem() -> str: + return rsa.generate_private_key(public_exponent=65537, key_size=2048).private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() + + +def _vertex_service_account_json(url: str) -> str: + return json.dumps( + { + "type": "service_account", + "project_id": "cc-scripted-project", + "private_key_id": "scripted", + "private_key": _vertex_private_key_pem(), + "client_email": "scripted@cc-scripted-project.iam.gserviceaccount.com", + "client_id": "0", + "auth_uri": f"{url}/_oauth/authorize", + "token_uri": f"{url}/_oauth/token", + } + ) + + +def register_scenario_deployment( + scenario: Scenario, + model: FrontierModel, + case: Case, + marker: str, +) -> str: + control_url: Final = os.environ["INTEGRATION_SCRIPTED_PROVIDER_URL"].rstrip("/") + sidecar_scenario: Final = case.scenario( + scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" + ) + handle: Final = register_scenario(sidecar_scenario) + scenario.cleanups.callback(delete_scenario, handle) + model_name: Final = f"{model.model_name}-{marker}" + parameters: Final = { + "model": model.litellm_model, + "api_key": model.api_key, + "api_base": handle.api_base(), + **model.litellm_params, + **( + {"vertex_credentials": _vertex_service_account_json(control_url)} + if model.wire == "vertex_generate" + else {} + ), + } + created: Final = scenario.gateway.post( + "/model/new", + JSON_OBJECT.validate_python({ + "model_name": model_name, + "litellm_params": parameters, + "model_info": {"base_model": model.base_model}, + }), + ) + identity: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, identity) + return model_name diff --git a/tests/e2e/cost_map.json b/tests/integration/cost_calculation/cost_map.json similarity index 100% rename from tests/e2e/cost_map.json rename to tests/integration/cost_calculation/cost_map.json diff --git a/tests/e2e/cost_calculation/cost_matrix.py b/tests/integration/cost_calculation/cost_matrix.py similarity index 98% rename from tests/e2e/cost_calculation/cost_matrix.py rename to tests/integration/cost_calculation/cost_matrix.py index 5e652421182..3c47cc16051 100644 --- a/tests/e2e/cost_calculation/cost_matrix.py +++ b/tests/integration/cost_calculation/cost_matrix.py @@ -2,9 +2,9 @@ the request/response cases from ``cases.json``, and the loaders both use. Two data files drive the suite; nothing in Python lists models or cases: -- ``tests/e2e/cost_map.json`` is the proxy's ENTIRE model cost map +- ``tests/integration/cost_calculation/cost_map.json`` is the proxy's ENTIRE model cost map (LITELLM_MODEL_COST_MAP_URL); every entry becomes a deployment under test. -- ``tests/e2e/cost_calculation/cases.json`` is the case list plus the reviewed +- ``tests/integration/cost_calculation/cases.json`` is the case list plus the reviewed goldens: each exact-spend case carries an ``expected`` cell per map key it runs against, each recount case carries its ``models`` list, so matrix membership and expected values are literal data read side by side. @@ -27,9 +27,9 @@ from types import MappingProxyType from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, TypeAdapter -from scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire +from integration._support.scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire -COST_MAP_PATH: Final = Path(__file__).resolve().parent.parent / "cost_map.json" +COST_MAP_PATH: Final = Path(__file__).resolve().parent / "cost_map.json" CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" class SearchContextCostPerQuery(BaseModel): @@ -506,7 +506,7 @@ VIDEO_INPUT_DATA_URL: Final = video_input_data_url() def matrix_data_errors() -> tuple[str, ...]: """Consistency findings for the data files, as human-readable strings. - Called at collection time by the e2e suite, so a map key named by a case + Called at collection time by the integration suite, so a map key named by a case but absent from cost_map.json fails the suite's collection loudly. """ unknown_deployments: Final = sorted( diff --git a/tests/integration/cost_calculation/test_token_pricing.py b/tests/integration/cost_calculation/test_token_pricing.py new file mode 100644 index 00000000000..29263b0a6c2 --- /dev/null +++ b/tests/integration/cost_calculation/test_token_pricing.py @@ -0,0 +1,223 @@ +"""Token pricing coverage for the integration scripted-provider cost shard.""" + +from __future__ import annotations + +import uuid +from typing import Final, cast + +import pytest +from pydantic import JsonValue + +from integration._support.client import JSON_OBJECT, Gateway +from integration._support.scripted_provider import ScriptedUsage, Wire +from integration.cost_calculation.conftest import ( + approx_equal, + assert_total_is_sum_of_components, + poll_cost_row, + register_scenario_deployment, +) +from integration.cost_calculation.cost_matrix import ( + AUDIO_INPUT_DATA_URL, + FRONTIER_MODELS, + IMAGE_INPUT_DATA_URL, + SERVICE_TIER_REQUEST_WIRES, + VIDEO_INPUT_DATA_URL, + Case, + FrontierModel, + cases_for, + matrix_data_errors, + recount_cost, +) + +if _data_errors := matrix_data_errors(): + raise ValueError("\n".join(_data_errors)) + +def _case_id(param: tuple[FrontierModel, Case]) -> str: + model, case = param + return f"{model.map_key.replace('/', '-')}-{case.name}" + + +_MATRIX: Final = tuple( + pytest.param( + (model, case), + marks=pytest.mark.covers( + "quota_management.spend_tracking.scripted_wire.logs_cost" + if case.family == "transport" + else "quota_management.spend_tracking.cost_matrix.logs_cost" + ), + id=_case_id((model, case)), + ) + for model in FRONTIER_MODELS + for case in cases_for(model) +) +_CACHE_WIRES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) +_WEB_SEARCH_OPTION_WIRES: Final = frozenset({"openai_chat", "azure_chat", "openai_responses"}) + + +def _cache_control(usage: ScriptedUsage, wire: Wire) -> dict[str, JsonValue] | None: + if wire not in _CACHE_WIRES: + return None + if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens): + return None + return {"type": "ephemeral", **({"ttl": "1h"} if usage.cache_write_1h_tokens else {})} + + +def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) -> dict[str, JsonValue]: + usage: Final = case.usage_for(model.map_key) + user_parts: Final = [ + {"type": "text", "text": f"{marker} summarize the attached material in one line and name the city weather"}, + *( + [{"type": "image_url", "image_url": {"url": IMAGE_INPUT_DATA_URL, "detail": "high"}}] + if case.image_input + else [] + ), + *( + [{"type": "input_audio", "input_audio": {"data": AUDIO_INPUT_DATA_URL.split(",", 1)[1], "format": "wav"}}] + if case.audio_input + else [] + ), + *( + [{"type": "file", "file": {"file_data": VIDEO_INPUT_DATA_URL, "format": "mp4"}}] + if case.video_input + else [] + ), + ] + tools: Final[list[JsonValue]] = [ + *( + [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + "days": {"type": "integer", "description": "Forecast horizon in days"}, + "units": {"type": "string", "enum": ["metric", "imperial"]}, + }, + "required": ["city"], + }, + }, + } + ] + if case.tool_call + else [] + ), + *( + [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] + if case.web_search is not None and model.wire == "anthropic_messages" + else [] + ), + *( + [{"googleSearch": {}}] + if case.web_search is not None and model.wire in ("gemini_generate", "vertex_generate") + else [] + ), + *([{"googleMaps": {}}] if case.google_maps else []), + *([{"type": "file_search", "vector_store_ids": ["vs_cost_calc_fixture"]}] if case.file_search else []), + ] + cache_control: Final = _cache_control(usage, model.wire) + message: Final = { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + **({"cache_control": cache_control} if cache_control else {}), + } + ], + } + return cast(dict[str, JsonValue], { + "model": model_name, + "messages": [message, {"role": "user", "content": user_parts}], + "stream": case.stream, + **({"stream_options": {"include_usage": True}} if case.stream else {}), + **( + {"service_tier": case.service_tier} + if case.service_tier is not None and model.wire in SERVICE_TIER_REQUEST_WIRES + else {} + ), + **({"reasoning_effort": "medium"} if case.reasoning else {}), + **( + {"modalities": ["text", "audio"] if case.audio_output else ["text"]} + if case.audio_input or case.audio_output + else {} + ), + **({"audio": {"voice": "alloy", "format": "pcm16"}} if case.audio_output else {}), + **( + {"web_search_options": {"search_context_size": case.web_search}} + if case.web_search is not None and model.wire in _WEB_SEARCH_OPTION_WIRES + else {} + ), + **({"tools": tools} if tools else {}), + **({"tool_choice": "auto"} if case.tool_call and model.wire != "bedrock_converse" else {}), + "allowed_openai_params": [ + name + for name, sent in ( + ("tool_choice", case.tool_call and model.wire != "bedrock_converse"), + ("modalities", case.audio_input or case.audio_output), + ("audio", case.audio_output), + ("web_search_options", case.web_search is not None), + ("reasoning_effort", case.reasoning), + ) + if sent + ], + }) + + +def _assert_stream_has_no_error(response_text: str) -> None: + for line in response_text.splitlines(): + if not line.startswith("data:"): + continue + payload = line.removeprefix("data:").strip() + if payload == "[DONE]": + continue + parsed = JSON_OBJECT.validate_json(payload) + assert "error" not in parsed, f"stream carried an error event: {parsed}" + + +@pytest.mark.parametrize("model_case", _MATRIX) +def test_scripted_usage_bills_at_map_rates( + gateway: Gateway, + model_case: tuple[FrontierModel, Case], +) -> None: + model, case = model_case + marker: Final = uuid.uuid4().hex[:12] + with gateway.scenario() as scenario: + key: Final = scenario.key() + model_name: Final = register_scenario_deployment(scenario, model, case, marker) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + _chat_body(model, case, model_name, marker), + key=key, + ) + assert response.is_success, ( + f"{model.map_key}/{case.name}: proxy returned {response.status_code}: {response.text[:400]}" + ) + if case.stream: + _assert_stream_has_no_error(response.text) + row: Final = poll_cost_row(key) + if not case.exact_spend: + assert row.prompt_tokens is not None and row.prompt_tokens > 0 + assert row.completion_tokens is not None and row.completion_tokens > 0 + if case.image_input: + assert row.prompt_tokens < 4000 + assert row.spend is not None and approx_equal( + row.spend, recount_cost(model, case, row.prompt_tokens, row.completion_tokens) + ) + assert_total_is_sum_of_components(row) + return + golden: Final = case.expected_for(model) + if not case.stream: + header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) + assert header is not None and approx_equal(float(header), golden.spend) + assert row.spend is not None and approx_equal(row.spend, golden.spend) + breakdown: Final = row.breakdown + assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, golden.input_cost) + assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, golden.output_cost) + assert row.prompt_tokens == golden.prompt_tokens + assert row.completion_tokens == golden.completion_tokens + assert_total_is_sum_of_components(row) From f836bb481df992b5b4987df8d2d3f734832c7171 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 00:19:11 +0000 Subject: [PATCH 22/30] test(integration): keep cost diagnostics and widen shard timeout Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- .circleci/scripts/run_integration.sh | 6 ++- tests/integration/README.md | 4 +- .../integration/cost_calculation/conftest.py | 12 +++-- .../cost_calculation/test_token_pricing.py | 50 +++++++++++++------ 5 files changed, 53 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e089436920..fa0d3f2c952 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2987,7 +2987,7 @@ jobs: - run: name: Run owned integration contracts command: bash .circleci/scripts/run_integration.sh << parameters.suite >> - no_output_timeout: 15m + no_output_timeout: 25m - run: name: Stop owned database and Redis when: always diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 17850bef4da..8194fb94bbc 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -9,6 +9,10 @@ fi suite="${1:?integration suite required}" results="test-results/integration-${suite}" mkdir -p "$results" +shard_timeout=11m +if [ "$suite" = cost ]; then + shard_timeout=20m +fi integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')" upstream_pid="" scripted_provider_pid="" @@ -181,7 +185,7 @@ if [ "$suite" = browser ]; then exit 0 fi -timeout --signal=TERM --kill-after=20s 11m env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ +timeout --signal=TERM --kill-after=20s "$shard_timeout" env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \ INTEGRATION_RUN_ID="$integration_identity" \ DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \ diff --git a/tests/integration/README.md b/tests/integration/README.md index 0049a640111..814d03a2875 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -4,7 +4,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local The `cost` group runs the scripted-provider cost matrix through a dedicated sidecar. The sidecar serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry -Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions` or `sdk` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate +Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate Management also requires `INTEGRATION_PEER_URL`, `REDIS_HOST` and `REDIS_PORT`. CircleCI starts two directly addressed proxy processes sharing only that job's stores. The test-only CLI wrapper supplies enterprise route entitlement, following the existing behavior suite's convention. It does not qualify license validation; run it with one worker and no reload @@ -22,7 +22,7 @@ Fixtures must contain synthetic data only. Keep private incident records and sou Database cases own their temporary schemas, roles, constraints and proxy processes. They prove reader-versus-writer execution with PostgreSQL lock observations, exercise real transaction wait limits and verify rollback after a reached database failure -Accounting cases compare persisted input and output cost components against literal rates, including zero and default prices. Cache state models assert actual upstream calls, response identity and every persisted charge. Generated accounting tests have a 180-second test limit to accommodate the asynchronous spend writer; CircleCI keeps the whole shard capped at 11 minutes +Accounting cases compare persisted input and output cost components against literal rates, including zero and default prices. Cache state models assert actual upstream calls, response identity and every persisted charge. Generated accounting tests have a 180-second test limit to accommodate the asynchronous spend writer; CircleCI keeps other shards capped at 11 minutes and gives the cost shard 20 minutes Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index bc08aa554f5..ab162725eef 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -54,14 +54,20 @@ def approx_equal(actual: float, expected: float) -> bool: return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) -def assert_total_is_sum_of_components(row: CostRow) -> None: +def assert_total_is_sum_of_components(row: CostRow, context: str) -> None: breakdown: Final = row.breakdown total: Final = sum( cost or 0.0 for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) ) - assert breakdown.total_cost is not None and approx_equal(breakdown.total_cost, total) - assert row.spend is not None and approx_equal(row.spend, breakdown.total_cost) + assert breakdown.total_cost is not None and approx_equal(breakdown.total_cost, total), ( + f"{context}: total_cost {breakdown.total_cost} != input_cost {breakdown.input_cost} " + f"+ output_cost {breakdown.output_cost} + tool_usage_cost {breakdown.tool_usage_cost} " + f"(sum {total})" + ) + assert row.spend is not None and approx_equal(row.spend, breakdown.total_cost), ( + f"{context}: row spend {row.spend} != breakdown total_cost {breakdown.total_cost}" + ) def _row(value: Mapping[str, object]) -> CostRow | None: diff --git a/tests/integration/cost_calculation/test_token_pricing.py b/tests/integration/cost_calculation/test_token_pricing.py index 29263b0a6c2..72510b03423 100644 --- a/tests/integration/cost_calculation/test_token_pricing.py +++ b/tests/integration/cost_calculation/test_token_pricing.py @@ -200,24 +200,46 @@ def test_scripted_usage_bills_at_map_rates( if case.stream: _assert_stream_has_no_error(response.text) row: Final = poll_cost_row(key) + context: Final = f"{model.map_key}/{case.name}" if not case.exact_spend: - assert row.prompt_tokens is not None and row.prompt_tokens > 0 - assert row.completion_tokens is not None and row.completion_tokens > 0 - if case.image_input: - assert row.prompt_tokens < 4000 - assert row.spend is not None and approx_equal( - row.spend, recount_cost(model, case, row.prompt_tokens, row.completion_tokens) + assert row.prompt_tokens is not None and row.prompt_tokens > 0, ( + f"{context}: no-usage stream counted no input tokens: prompt_tokens={row.prompt_tokens}" ) - assert_total_is_sum_of_components(row) + assert row.completion_tokens is not None and row.completion_tokens > 0, ( + f"{context}: no-usage stream counted no output tokens: completion_tokens={row.completion_tokens}" + ) + if case.image_input: + assert row.prompt_tokens < 4000, ( + f"{context}: image data URL looks tokenized as text: prompt_tokens={row.prompt_tokens}" + ) + recount: Final = recount_cost(model, case, row.prompt_tokens, row.completion_tokens) + assert row.spend is not None and approx_equal( + row.spend, recount + ), f"{context}: no-usage stream spend {row.spend} != recount {recount} at map rates" + assert_total_is_sum_of_components(row, context) return golden: Final = case.expected_for(model) if not case.stream: header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) - assert header is not None and approx_equal(float(header), golden.spend) - assert row.spend is not None and approx_equal(row.spend, golden.spend) + assert header is not None and approx_equal(float(header), golden.spend), ( + f"{context}: x-litellm-response-cost {header} != golden {golden.spend}" + ) + assert row.spend is not None and approx_equal(row.spend, golden.spend), ( + f"{context}: spend {row.spend} != golden {golden.spend} " + f"(breakdown {row.breakdown.model_dump()})" + ) breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, golden.input_cost) - assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, golden.output_cost) - assert row.prompt_tokens == golden.prompt_tokens - assert row.completion_tokens == golden.completion_tokens - assert_total_is_sum_of_components(row) + assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, golden.input_cost), ( + f"{context}: gross input_cost {breakdown.input_cost} != golden {golden.input_cost}; " + "cached/written tokens billed at the input rate" + ) + assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, golden.output_cost), ( + f"{context}: output_cost {breakdown.output_cost} != golden {golden.output_cost}" + ) + assert row.prompt_tokens == golden.prompt_tokens, ( + f"{context}: prompt_tokens {row.prompt_tokens} != golden {golden.prompt_tokens}" + ) + assert row.completion_tokens == golden.completion_tokens, ( + f"{context}: completion_tokens {row.completion_tokens} != golden {golden.completion_tokens}" + ) + assert_total_is_sum_of_components(row, context) From e52eea84e6f1aa34fcc21d434118b44ff39e711b Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 00:50:26 +0000 Subject: [PATCH 23/30] test(integration): serve scripted wires from the shared upstream Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/scripts/run_integration.sh | 26 +--- .../scripts/wait_integration_services.py | 5 - tests/integration/README.md | 4 +- tests/integration/_support/scripted_client.py | 10 +- ...scripted_provider.py => scripted_wires.py} | 114 ++---------------- tests/integration/_support/upstream.py | 85 ++++++++++++- .../integration/cost_calculation/conftest.py | 2 +- .../cost_calculation/cost_matrix.py | 2 +- .../cost_calculation/test_token_pricing.py | 4 +- 9 files changed, 107 insertions(+), 145 deletions(-) rename tests/integration/_support/{scripted_provider.py => scripted_wires.py} (91%) diff --git a/.circleci/scripts/run_integration.sh b/.circleci/scripts/run_integration.sh index 8194fb94bbc..501bf68b7ca 100644 --- a/.circleci/scripts/run_integration.sh +++ b/.circleci/scripts/run_integration.sh @@ -10,12 +10,8 @@ suite="${1:?integration suite required}" results="test-results/integration-${suite}" mkdir -p "$results" shard_timeout=11m -if [ "$suite" = cost ]; then - shard_timeout=20m -fi integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')" upstream_pid="" -scripted_provider_pid="" proxy_pid="" peer_pid="" launched_pid="" @@ -27,9 +23,9 @@ cleanup() { original_status=$? trap - EXIT INT TERM sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \ - "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" "$scripted_provider_pid" \ + "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \ > "$results/process-cleanup.txt" 2>&1 || original_status=1 - for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid" "$scripted_provider_pid"; do + for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do if [ -n "$owned_pid" ]; then kill -- "-$owned_pid" 2>/dev/null || true for _ in {1..50}; do @@ -74,7 +70,6 @@ export STORE_MODEL_IN_DB=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 export INTEGRATION_PROXY_URL=http://127.0.0.1:4000 export INTEGRATION_PEER_URL="" export INTEGRATION_UPSTREAM_URL=http://127.0.0.1:8190 -export INTEGRATION_SCRIPTED_PROVIDER_URL="" export INTEGRATION_MASTER_KEY="$LITELLM_MASTER_KEY" export LITELLM_UI_PATH="$PWD/litellm/proxy/_experimental/out" if [ "$suite" = browser ]; then @@ -115,18 +110,7 @@ setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 & upstream_pid=$! if [ "$suite" = cost ]; then - export INTEGRATION_SCRIPTED_PROVIDER_URL=http://127.0.0.1:8191 - setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \ - .venv/bin/python -m integration._support.scripted_provider --port 8191 \ - > "$results/scripted-provider.log" 2>&1 & - scripted_provider_pid=$! - for _ in {1..90}; do - if curl --noproxy '*' -fsS "$INTEGRATION_SCRIPTED_PROVIDER_URL/health" >/dev/null 2>&1; then - break - fi - sleep 1 - done - curl --noproxy '*' -fsS "$INTEGRATION_SCRIPTED_PROVIDER_URL/health" >/dev/null + export INTEGRATION_WORKERS=8 fi start_proxy() { local port="$1" @@ -134,7 +118,7 @@ start_proxy() { local -a cost_map_env if [ "$suite" = cost ]; then cost_map_env=( - "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_SCRIPTED_PROVIDER_URL/_cost_map" + "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_UPSTREAM_URL/_cost_map" "MODEL_COST_MAP_MIN_MODEL_COUNT=1" "MODEL_COST_MAP_MAX_SHRINK_RATIO=0" ) @@ -190,7 +174,7 @@ timeout --signal=TERM --kill-after=20s "$shard_timeout" env -i PATH="$PATH" HOME DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \ INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \ INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \ - INTEGRATION_SCRIPTED_PROVIDER_URL="$INTEGRATION_SCRIPTED_PROVIDER_URL" \ + INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \ INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \ INTEGRATION_SEED="$INTEGRATION_SEED" \ INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \ diff --git a/.circleci/scripts/wait_integration_services.py b/.circleci/scripts/wait_integration_services.py index 462874e8aa6..486e37cba00 100644 --- a/.circleci/scripts/wait_integration_services.py +++ b/.circleci/scripts/wait_integration_services.py @@ -9,7 +9,6 @@ from redis import Redis def main() -> None: primary: Final = os.environ["INTEGRATION_PROXY_URL"] peer: Final = os.environ.get("INTEGRATION_PEER_URL") - scripted_provider: Final = os.environ.get("INTEGRATION_SCRIPTED_PROVIDER_URL") or None proxies: Final = (primary, peer) if peer else (primary,) deadline: Final = time.monotonic() + 90 headers: Final = {"Authorization": f"Bearer {os.environ['INTEGRATION_MASTER_KEY']}"} @@ -20,10 +19,6 @@ def main() -> None: try: ready: Final = ( client.get(f"{os.environ['INTEGRATION_UPSTREAM_URL']}/health").status_code == 200 - and ( - scripted_provider is None - or client.get(f"{scripted_provider}/health").status_code == 200 - ) and all(client.get(f"{url}/health/readiness").status_code == 200 for url in proxies) ) if ready: diff --git a/tests/integration/README.md b/tests/integration/README.md index 814d03a2875..49b413b17c5 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The `cost` group runs the scripted-provider cost matrix through a dedicated sidecar. The sidecar serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry +The `cost` group runs the scripted-wire cost matrix through the shared integration upstream. The upstream serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate @@ -22,7 +22,7 @@ Fixtures must contain synthetic data only. Keep private incident records and sou Database cases own their temporary schemas, roles, constraints and proxy processes. They prove reader-versus-writer execution with PostgreSQL lock observations, exercise real transaction wait limits and verify rollback after a reached database failure -Accounting cases compare persisted input and output cost components against literal rates, including zero and default prices. Cache state models assert actual upstream calls, response identity and every persisted charge. Generated accounting tests have a 180-second test limit to accommodate the asynchronous spend writer; CircleCI keeps other shards capped at 11 minutes and gives the cost shard 20 minutes +Accounting cases compare persisted input and output cost components against literal rates, including zero and default prices. Cache state models assert actual upstream calls, response identity and every persisted charge. Generated accounting tests have a 180-second test limit to accommodate the asynchronous spend writer; CircleCI keeps the whole shard capped at 11 minutes Provider contracts exercise actual TCP requests with synthetic credentials and local protocol peers. The S3 verifier uses independently implemented equations, a published known-answer vector, a fixed signing clock and deliberately invalid signed requests. Bedrock cases clear ambient AWS credential sources and check the literal model path, loaded role references, STS requests and bearer-only behavior diff --git a/tests/integration/_support/scripted_client.py b/tests/integration/_support/scripted_client.py index 7818488fae0..9502740b1b5 100644 --- a/tests/integration/_support/scripted_client.py +++ b/tests/integration/_support/scripted_client.py @@ -1,4 +1,4 @@ -"""Client for registering scenarios with the integration scripted provider.""" +"""Client for registering scenarios with the integration upstream.""" from __future__ import annotations @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import Final import httpx -from integration._support.scripted_provider import ( +from integration._support.scripted_wires import ( WIRE_MOUNTS, Scenario, ScenarioDeleted, @@ -15,7 +15,7 @@ from integration._support.scripted_provider import ( Wire, ) -CONTROL_URL: Final = os.environ.get("INTEGRATION_SCRIPTED_PROVIDER_URL", "http://127.0.0.1:8191").rstrip("/") +CONTROL_URL: Final = os.environ.get("INTEGRATION_UPSTREAM_URL", "http://127.0.0.1:8190").rstrip("/") @dataclass(frozen=True, slots=True) @@ -33,7 +33,7 @@ class ScenarioHandle: def register_scenario(scenario: Scenario) -> ScenarioHandle: response: Final = httpx.post( - f"{CONTROL_URL}/_scenarios", + f"{CONTROL_URL}/__scenarios", json=scenario.model_dump(mode="json"), trust_env=False, timeout=15, @@ -49,7 +49,7 @@ def register_scenario(scenario: Scenario) -> ScenarioHandle: def delete_scenario(handle: ScenarioHandle) -> None: response: Final = httpx.delete( - f"{CONTROL_URL}/_scenarios/{handle.scenario_id}", + f"{CONTROL_URL}/__scenarios/{handle.scenario_id}", trust_env=False, timeout=15, ) diff --git a/tests/integration/_support/scripted_provider.py b/tests/integration/_support/scripted_wires.py similarity index 91% rename from tests/integration/_support/scripted_provider.py rename to tests/integration/_support/scripted_wires.py index d5e0fd7e9cf..ae5ed3abd61 100644 --- a/tests/integration/_support/scripted_provider.py +++ b/tests/integration/_support/scripted_wires.py @@ -1,22 +1,17 @@ -"""Scripted provider sidecar for the cost-calculation integration suite. +"""Scripted provider wires for the cost-calculation integration suite. -A standalone process (``python -m integration._support.scripted_provider``) that -pretends to be an LLM provider for the proxy under test. The suite registers a -Scenario over a small control API; the provider wire routes then answer the -proxy's upstream calls with the scripted usage figures, in the exact wire shape +The shared integration upstream registers a Scenario over a small control API; +the provider wire routes answer the proxy's upstream calls with the scripted usage figures, in the exact wire shape the real provider would emit (OpenAI chat completions, OpenAI Responses, Anthropic Messages, Gemini generateContent, or the OpenAI-compatible Together / Fireworks surfaces). Because the usage is scripted, expected spend is literal arithmetic on the test cost map's rates, with no dependency on what a real provider would report. -Layout on one port: +The upstream exposes: -- ``GET /health`` liveness -- ``POST /_scenarios`` register a Scenario JSON, returns its id -- ``DELETE /_scenarios/`` remove it -- ``POST /_oauth/token`` fake Google OAuth token endpoint for the - Vertex service-account credential's refresh call +- ``POST /__scenarios`` register a Scenario JSON, returns its id +- ``DELETE /__scenarios/`` remove it - ``POST ///`` provider wire; mount is one of ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks``, ``azure``, ``bedrock``, ``vertex`` and the remainder is whatever path the provider @@ -32,22 +27,18 @@ final stream chunk carries usage or the provider reports none. from __future__ import annotations -import argparse import json import struct -import sys import threading import time import zlib from collections.abc import Mapping from dataclasses import dataclass -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path from types import MappingProxyType -from typing import Final, Literal, TypeAlias, cast +from typing import Final, Literal, TypeAlias from urllib.parse import unquote, urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError, model_validator +from pydantic import BaseModel, ConfigDict, TypeAdapter, model_validator Wire: TypeAlias = Literal[ "openai_chat", @@ -1307,7 +1298,7 @@ def _render( # ---------- registry + request routing ---------- -class _ScenarioStore: +class ScenarioStore: def __init__(self) -> None: self._lock: Final = threading.Lock() self._scenarios: dict[str, Scenario] = {} # mutable-ok: server state, guarded by _lock @@ -1359,55 +1350,9 @@ def _request_model(body: bytes, path_tail: str, scenario: Scenario) -> str: return scenario.model -def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: +def render(store: ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: path: Final = urlsplit(raw_path).path segments: Final = tuple(segment for segment in path.split("/") if segment) - if method == "GET" and segments == ("health",): - return RenderedResponse(200, "application/json", _json_bytes(_jobj(("status", "ok")))) - if method == "GET" and segments == ("_cost_map",): - return RenderedResponse( - 200, - "application/json", - (Path(__file__).resolve().parents[1] / "cost_calculation" / "cost_map.json").read_bytes(), - ) - if segments and segments[0] == "_oauth": - if method == "POST" and segments == ("_oauth", "token"): - return RenderedResponse( - 200, - "application/json", - _json_bytes( - _jobj( - ("access_token", "scripted-token"), - ("token_type", "Bearer"), - ("expires_in", 3600), - ) - ), - ) - return RenderedResponse( - 404, "application/json", _json_bytes(_jobj(("error", "unknown control route"))) - ) - if segments and segments[0] == "_scenarios": - if method == "POST" and len(segments) == 1: - try: - scenario: Final = Scenario.model_validate_json(body) - except ValidationError as exc: - return RenderedResponse( - 400, "application/json", _json_bytes(_jobj(("error", str(exc)))) - ) - store.put(scenario) - return RenderedResponse( - 200, "application/json", _json_bytes(_jobj(("scenario_id", scenario.scenario_id))) - ) - if method == "DELETE" and len(segments) == 2: - deleted: Final = store.drop(segments[1]) - return RenderedResponse( - 200 if deleted else 404, - "application/json", - _json_bytes(_jobj(("deleted", deleted))), - ) - return RenderedResponse( - 404, "application/json", _json_bytes(_jobj(("error", "unknown control route"))) - ) if len(segments) < 2 or method != "POST": return RenderedResponse( 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) @@ -1441,42 +1386,3 @@ def handle_request(store: _ScenarioStore, method: str, raw_path: str, body: byte requested_model=_request_model(body, tail, found), path_tail=tail, ) - - -class _ScriptedHandler(BaseHTTPRequestHandler): - store: Final[_ScenarioStore] = _ScenarioStore() - - def _dispatch(self, method: str) -> None: - length: Final = int(self.headers.get("content-length") or 0) - body: Final = self.rfile.read(length) if length else b"" - rendered: Final = handle_request(self.store, method, self.path, body) - self.send_response(rendered.status_code) - self.send_header("content-type", rendered.content_type) - self.send_header("content-length", str(len(rendered.body))) - self.end_headers() - self.wfile.write(rendered.body) - - def do_GET(self) -> None: - self._dispatch("GET") - - def do_POST(self) -> None: - self._dispatch("POST") - - def do_DELETE(self) -> None: - self._dispatch("DELETE") - - - -DEFAULT_PORT: Final = 8191 - - -def serve(port: int = DEFAULT_PORT, bind_host: str = "127.0.0.1") -> None: - server: Final = ThreadingHTTPServer((bind_host, port), _ScriptedHandler) - sys.stderr.write(f"scripted-provider listening on http://{bind_host}:{port}\n") - server.serve_forever() - - -if __name__ == "__main__": - parser: Final = argparse.ArgumentParser() - parser.add_argument("--port", type=int, default=8191) - serve(port=cast(int, parser.parse_args().port)) diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index 04a6ea02eec..c8e77ad513a 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -1,19 +1,22 @@ from __future__ import annotations import argparse -from dataclasses import dataclass, field from collections import deque +import json +from dataclasses import dataclass, field +from pathlib import Path from queue import SimpleQueue -from typing import Final +from typing import Final, cast import uvicorn -from pydantic import JsonValue, TypeAdapter +from pydantic import JsonValue, TypeAdapter, ValidationError from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import JSONResponse, Response from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations +from integration._support.scripted_wires import RenderedResponse, Scenario, ScenarioStore, render JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) INTERNAL_FIELDS: Final = frozenset( @@ -48,6 +51,7 @@ class Observation: class Provider: observations: SimpleQueue[Observation] = field(default_factory=SimpleQueue) scripts: dict[str, deque[int]] = field(default_factory=dict) + scenario_store: ScenarioStore = field(default_factory=ScenarioStore) async def chat(self, request: Request) -> Response: body: Final = JSON_OBJECT.validate_json(await request.body()) @@ -103,16 +107,89 @@ class Provider: } ) + async def register_scenario(self, request: Request) -> Response: + try: + scenario: Final = Scenario.model_validate_json(await request.body()) + except ValidationError as exc: + return self._render( + RenderedResponse(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) + ) + self.scenario_store.put(scenario) + return self._render( + RenderedResponse( + 200, + "application/json", + json.dumps({"scenario_id": scenario.scenario_id}).encode("utf-8"), + ) + ) + + async def delete_scenario(self, request: Request) -> Response: + scenario_id: Final = cast(str, request.path_params["scenario_id"]) + deleted: Final = self.scenario_store.drop(scenario_id) + return self._render( + RenderedResponse( + 200 if deleted else 404, + "application/json", + json.dumps({"deleted": deleted}).encode("utf-8"), + ) + ) + + async def cost_map(self, _request: Request) -> Response: + return self._render( + RenderedResponse( + 200, + "application/json", + (Path(__file__).resolve().parents[1] / "cost_calculation" / "cost_map.json").read_bytes(), + ) + ) + + async def oauth_token(self, _request: Request) -> Response: + return self._render( + RenderedResponse( + 200, + "application/json", + json.dumps( + { + "access_token": "scripted-token", + "token_type": "Bearer", + "expires_in": 3600, + } + ).encode("utf-8"), + ) + ) + + async def scripted(self, request: Request) -> Response: + rendered: Final = render( + self.scenario_store, + request.method, + request.url.path, + await request.body(), + ) + return self._render(rendered) + + @staticmethod + def _render(rendered: RenderedResponse) -> Response: + return Response( + content=rendered.body, + status_code=rendered.status_code, + media_type=rendered.content_type, + ) + def app(self) -> Starlette: return Starlette( routes=[ Route("/health", health), Route("/__observations", self.observed), Route("/__scripts/{model}", self.script, methods=["POST", "DELETE", "GET"]), + Route("/__scenarios", self.register_scenario, methods=["POST"]), + Route("/__scenarios/{scenario_id}", self.delete_scenario, methods=["DELETE"]), + Route("/_cost_map", self.cost_map, methods=["GET"]), + Route("/_oauth/token", self.oauth_token, methods=["POST"]), Route("/v1/chat/completions", self.chat, methods=["POST"]), Route("/v1/completions", completions, methods=["POST"]), Route("/v1/embeddings", embeddings, methods=["POST"]), Route("/v1/moderations", moderations, methods=["POST"]), + Route("/{scenario_id}/{tail:path}", self.scripted, methods=["POST"]), ] ) @@ -121,7 +198,7 @@ def main() -> None: parser: Final = argparse.ArgumentParser() parser.add_argument("--port", type=int, default=8190) arguments: Final = parser.parse_args() - uvicorn.run(Provider().app(), host="127.0.0.1", port=arguments.port, access_log=False) + uvicorn.run(Provider().app(), host="127.0.0.1", port=cast(int, arguments.port), access_log=False) if __name__ == "__main__": diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index ab162725eef..66eb373df33 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -122,7 +122,7 @@ def register_scenario_deployment( case: Case, marker: str, ) -> str: - control_url: Final = os.environ["INTEGRATION_SCRIPTED_PROVIDER_URL"].rstrip("/") + control_url: Final = os.environ["INTEGRATION_UPSTREAM_URL"].rstrip("/") sidecar_scenario: Final = case.scenario( scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" ) diff --git a/tests/integration/cost_calculation/cost_matrix.py b/tests/integration/cost_calculation/cost_matrix.py index 3c47cc16051..8b9e0aa9424 100644 --- a/tests/integration/cost_calculation/cost_matrix.py +++ b/tests/integration/cost_calculation/cost_matrix.py @@ -27,7 +27,7 @@ from types import MappingProxyType from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, TypeAdapter -from integration._support.scripted_provider import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire +from integration._support.scripted_wires import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire COST_MAP_PATH: Final = Path(__file__).resolve().parent / "cost_map.json" CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" diff --git a/tests/integration/cost_calculation/test_token_pricing.py b/tests/integration/cost_calculation/test_token_pricing.py index 72510b03423..69e2ac7ca0c 100644 --- a/tests/integration/cost_calculation/test_token_pricing.py +++ b/tests/integration/cost_calculation/test_token_pricing.py @@ -1,4 +1,4 @@ -"""Token pricing coverage for the integration scripted-provider cost shard.""" +"""Token pricing coverage for the integration scripted-wire cost shard.""" from __future__ import annotations @@ -9,7 +9,7 @@ import pytest from pydantic import JsonValue from integration._support.client import JSON_OBJECT, Gateway -from integration._support.scripted_provider import ScriptedUsage, Wire +from integration._support.scripted_wires import ScriptedUsage, Wire from integration.cost_calculation.conftest import ( approx_equal, assert_total_is_sum_of_components, From 6eb67a84235df6be9ccb84dce82e21f50d6c3cc2 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 00:50:29 +0000 Subject: [PATCH 24/30] test(integration): run the cost shard with xdist workers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- tests/integration/conftest.py | 36 ++++++++++++++++++++++++----------- tests/integration/run.py | 6 ++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fa0d3f2c952..6e089436920 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2987,7 +2987,7 @@ jobs: - run: name: Run owned integration contracts command: bash .circleci/scripts/run_integration.sh << parameters.suite >> - no_output_timeout: 25m + no_output_timeout: 15m - run: name: Stop owned database and Redis when: always diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 342952d44d4..f66ff7e74df 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,10 +1,11 @@ from __future__ import annotations import json -import os import hashlib +import os +from collections.abc import Sequence +from collections.abc import Iterator from importlib.metadata import version -from collections.abc import Generator, Iterator from pathlib import Path from typing import Final @@ -28,6 +29,26 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line("markers", "integration: owned real-service integration contracts") config.addinivalue_line("markers", "covers(*ids): independently asserted behavior contracts") config.stash[REPORTS] = [] + config.pluginmanager.register(IntegrationReportPlugin(config)) + + +class IntegrationReportPlugin: + def __init__(self, config: pytest.Config) -> None: + self.config = config + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.config.stash[REPORTS].append(report) + + @pytest.hookimpl(optionalhook=True) + def pytest_xdist_node_collection_finished(self, node: object, ids: Sequence[str]) -> None: + owned_prefix: Final = "tests/integration/" + self.config.stash[COLLECTED] = tuple( + nodeid + for nodeid in ids + if nodeid.split("::", 1)[0].startswith(owned_prefix) + and len(Path(nodeid.split("::", 1)[0]).parts) > 2 + and Path(nodeid.split("::", 1)[0]).parts[2] in OWNED_DIRECTORIES + ) def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: @@ -54,16 +75,9 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item config.stash[COLLECTED] = tuple(item.nodeid for item in owned) -@pytest.hookimpl(wrapper=True) -def pytest_runtest_makereport( - item: pytest.Item, call: pytest.CallInfo[None] -) -> Generator[None, pytest.TestReport, pytest.TestReport]: - report: Final = yield - item.config.stash[REPORTS].append(report) - return report - - def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + if hasattr(session.config, "workerinput"): + return destination: Final = os.environ.get("INTEGRATION_RESULTS_DIR") if destination is None: return diff --git a/tests/integration/run.py b/tests/integration/run.py index 759644f6ab6..f45164c5ca4 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -18,6 +18,7 @@ def main() -> int: parser.add_argument("--results", type=Path, default=Path("test-results/integration")) parser.add_argument("--seed", type=int, default=int(os.environ.get("INTEGRATION_SEED", "4106601"))) parser.add_argument("--order-seed", type=int, default=int(os.environ.get("INTEGRATION_ORDER_SEED", "0"))) + parser.add_argument("--workers", type=int, default=int(os.environ.get("INTEGRATION_WORKERS", "1"))) options: Final = parser.parse_args() root: Final = Path(__file__).resolve().parents[2] selected: Final = tuple( @@ -56,6 +57,11 @@ def main() -> int: f"--hypothesis-seed={options.seed}", f"--integration-order-seed={options.order_seed}", f"--junitxml={output / 'junit.xml'}", + *( + ("-n", str(options.workers)) + if options.workers > 1 + else () + ), ], cwd=root, env=environment, From a15b0fa6d2302d3ef86ddedb1857d4742b6af0dd Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 00:54:45 +0000 Subject: [PATCH 25/30] test(integration): tidy xdist collection bookkeeping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/conftest.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index f66ff7e74df..c54197c15e6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,21 +1,20 @@ from __future__ import annotations -import json import hashlib +import json import os -from collections.abc import Sequence -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from importlib.metadata import version from pathlib import Path from typing import Final -import pytest import httpx +import pytest from redis import Redis from tests.integration._support.client import Gateway, eventually, gateway_from_environment -from tests.integration._support.manifest import OWNED_DIRECTORIES, contracts from tests.integration._support.generation import LIFECYCLE_SETTINGS +from tests.integration._support.manifest import OWNED_DIRECTORIES, contracts COLLECTED: Final = pytest.StashKey[tuple[str, ...]]() REPORTS: Final = pytest.StashKey[list[pytest.TestReport]]() @@ -41,14 +40,12 @@ class IntegrationReportPlugin: @pytest.hookimpl(optionalhook=True) def pytest_xdist_node_collection_finished(self, node: object, ids: Sequence[str]) -> None: - owned_prefix: Final = "tests/integration/" - self.config.stash[COLLECTED] = tuple( - nodeid - for nodeid in ids - if nodeid.split("::", 1)[0].startswith(owned_prefix) - and len(Path(nodeid.split("::", 1)[0]).parts) > 2 - and Path(nodeid.split("::", 1)[0]).parts[2] in OWNED_DIRECTORIES - ) + self.config.stash[COLLECTED] = tuple(nodeid for nodeid in ids if _owned(nodeid)) + + +def _owned(nodeid: str) -> bool: + parts: Final = Path(nodeid.split("::", 1)[0]).parts + return parts[:2] == ("tests", "integration") and len(parts) > 3 and parts[2] in OWNED_DIRECTORIES def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: From 77f6166c392dc2fede07e79c0ef4af91ce9b01ad Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 01:30:24 +0000 Subject: [PATCH 26/30] test(integration): fold scenario client into upstream module Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/scripted_client.py | 57 ------------------- tests/integration/_support/upstream.py | 55 +++++++++++++++++- .../integration/cost_calculation/conftest.py | 2 +- 3 files changed, 55 insertions(+), 59 deletions(-) delete mode 100644 tests/integration/_support/scripted_client.py diff --git a/tests/integration/_support/scripted_client.py b/tests/integration/_support/scripted_client.py deleted file mode 100644 index 9502740b1b5..00000000000 --- a/tests/integration/_support/scripted_client.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Client for registering scenarios with the integration upstream.""" - -from __future__ import annotations - -import os -from dataclasses import dataclass -from typing import Final - -import httpx -from integration._support.scripted_wires import ( - WIRE_MOUNTS, - Scenario, - ScenarioDeleted, - ScenarioRegistered, - Wire, -) - -CONTROL_URL: Final = os.environ.get("INTEGRATION_UPSTREAM_URL", "http://127.0.0.1:8190").rstrip("/") - - -@dataclass(frozen=True, slots=True) -class ScenarioHandle: - scenario_id: str - wire: Wire - control_url: str - - def api_base(self) -> str: - return f"{self.control_url}/{self.scenario_id}/{self._mount()}" - - def _mount(self) -> str: - return WIRE_MOUNTS[self.wire] - - -def register_scenario(scenario: Scenario) -> ScenarioHandle: - response: Final = httpx.post( - f"{CONTROL_URL}/__scenarios", - json=scenario.model_dump(mode="json"), - trust_env=False, - timeout=15, - ) - response.raise_for_status() - result: Final = ScenarioRegistered.model_validate_json(response.content) - return ScenarioHandle( - scenario_id=result.scenario_id, - wire=scenario.wire, - control_url=CONTROL_URL, - ) - - -def delete_scenario(handle: ScenarioHandle) -> None: - response: Final = httpx.delete( - f"{CONTROL_URL}/__scenarios/{handle.scenario_id}", - trust_env=False, - timeout=15, - ) - response.raise_for_status() - ScenarioDeleted.model_validate_json(response.content) diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index c8e77ad513a..b3e6336dcee 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -4,10 +4,12 @@ import argparse from collections import deque import json from dataclasses import dataclass, field +import os from pathlib import Path from queue import SimpleQueue from typing import Final, cast +import httpx import uvicorn from pydantic import JsonValue, TypeAdapter, ValidationError from starlette.applications import Starlette @@ -16,7 +18,16 @@ from starlette.responses import JSONResponse, Response from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations -from integration._support.scripted_wires import RenderedResponse, Scenario, ScenarioStore, render +from integration._support.scripted_wires import ( + WIRE_MOUNTS, + RenderedResponse, + Scenario, + ScenarioDeleted, + ScenarioRegistered, + ScenarioStore, + Wire, + render, +) JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) INTERNAL_FIELDS: Final = frozenset( @@ -194,6 +205,48 @@ class Provider: ) +CONTROL_URL: Final = os.environ.get("INTEGRATION_UPSTREAM_URL", "http://127.0.0.1:8190").rstrip("/") + + +@dataclass(frozen=True, slots=True) +class ScenarioHandle: + scenario_id: str + wire: Wire + control_url: str + + def api_base(self) -> str: + return f"{self.control_url}/{self.scenario_id}/{self._mount()}" + + def _mount(self) -> str: + return WIRE_MOUNTS[self.wire] + + +def register_scenario(scenario: Scenario) -> ScenarioHandle: + response: Final = httpx.post( + f"{CONTROL_URL}/__scenarios", + json=scenario.model_dump(mode="json"), + trust_env=False, + timeout=15, + ) + response.raise_for_status() + result: Final = ScenarioRegistered.model_validate_json(response.content) + return ScenarioHandle( + scenario_id=result.scenario_id, + wire=scenario.wire, + control_url=CONTROL_URL, + ) + + +def delete_scenario(handle: ScenarioHandle) -> None: + response: Final = httpx.delete( + f"{CONTROL_URL}/__scenarios/{handle.scenario_id}", + trust_env=False, + timeout=15, + ) + response.raise_for_status() + ScenarioDeleted.model_validate_json(response.content) + + def main() -> None: parser: Final = argparse.ArgumentParser() parser.add_argument("--port", type=int, default=8190) diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 66eb373df33..0cbc837c184 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -13,7 +13,7 @@ from pydantic import BaseModel, ConfigDict from integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value from integration._support.database import read_rows -from integration._support.scripted_client import delete_scenario, register_scenario +from integration._support.upstream import delete_scenario, register_scenario from integration.cost_calculation.cost_matrix import Case, FrontierModel From 6ccba7fdb51592cbd56a38b000499f5eef75f86b Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 01:41:19 +0000 Subject: [PATCH 27/30] test(integration): drive scripted wires and provider wiring from data Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 2 +- tests/integration/_support/scripted_wires.py | 172 +++++++----------- tests/integration/_support/upstream.py | 4 +- tests/integration/_support/wires.json | 119 ++++++++++++ tests/integration/cost_calculation/cases.json | 74 ++++++++ .../cost_calculation/cost_matrix.py | 94 +++++----- .../cost_calculation/test_token_pricing.py | 10 +- 7 files changed, 317 insertions(+), 158 deletions(-) create mode 100644 tests/integration/_support/wires.json diff --git a/tests/integration/README.md b/tests/integration/README.md index 49b413b17c5..a007eb6dc68 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The `cost` group runs the scripted-wire cost matrix through the shared integration upstream. The upstream serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry +The `cost` group runs the scripted-wire cost matrix through the shared integration upstream. A provider speaking an existing response shape is a `wires.json` row, a `cases.json` `providers` row and cost-map entries; a new response shape needs a renderer in `scripted_wires.py` Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate diff --git a/tests/integration/_support/scripted_wires.py b/tests/integration/_support/scripted_wires.py index ae5ed3abd61..8da2c57c9a0 100644 --- a/tests/integration/_support/scripted_wires.py +++ b/tests/integration/_support/scripted_wires.py @@ -34,100 +34,26 @@ import time import zlib from collections.abc import Mapping from dataclasses import dataclass +from pathlib import Path from types import MappingProxyType -from typing import Final, Literal, TypeAlias +from typing import Final, Literal, TypeAlias, assert_never from urllib.parse import unquote, urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, model_validator -Wire: TypeAlias = Literal[ +Wire: TypeAlias = str +Shape: TypeAlias = Literal[ "openai_chat", "openai_responses", "anthropic_messages", "gemini_generate", - "together_chat", - "fireworks_chat", - "azure_chat", "bedrock_converse", - "vertex_generate", ] - -WIRE_MOUNTS: Final[Mapping[str, str]] = MappingProxyType( - { - "openai_chat": "openai", - "openai_responses": "openai", - "anthropic_messages": "anthropic", - "gemini_generate": "gemini", - "together_chat": "together", - "fireworks_chat": "fireworks", - "azure_chat": "azure", - "bedrock_converse": "bedrock", - "vertex_generate": "vertex", - } -) - StreamUsage: TypeAlias = Literal["final_chunk", "absent"] ServiceTier: TypeAlias = Literal["flex", "priority"] TerminalKind: TypeAlias = Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] -# Which terminal variant each wire can represent. -_TERMINAL_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( - { - "openai_responses": frozenset({"incomplete", "unvalidated"}), - "gemini_generate": frozenset({"prompt_blocked"}), - "vertex_generate": frozenset({"prompt_blocked"}), - } -) - - _BASE_USAGE_FIELDS: Final = frozenset({"fresh_input_tokens", "output_tokens"}) -_OPENAI_FAMILY_USAGE: Final = frozenset( - { - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "web_search_calls", - } -) -_CACHE_WRITE_USAGE: Final = frozenset({"cache_write_5m_tokens", "cache_write_1h_tokens"}) -_GEMINI_USAGE: Final = frozenset( - { - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "image_input_tokens", - "video_input_tokens", - "web_search_calls", - "google_maps_calls", - } -) - -_USAGE_CAPS: Final[Mapping[str, frozenset[str]]] = MappingProxyType( - { - wire: usage - for wire, usage in ( - ("openai_chat", _OPENAI_FAMILY_USAGE), - ("azure_chat", _OPENAI_FAMILY_USAGE), - ("together_chat", _OPENAI_FAMILY_USAGE), - ("fireworks_chat", _OPENAI_FAMILY_USAGE), - ( - "openai_responses", - frozenset( - {"cache_read_tokens", "reasoning_tokens", "web_search_calls", "file_search_calls"} - ), - ), - ( - "anthropic_messages", - frozenset({"cache_read_tokens", "web_search_calls"}) | _CACHE_WRITE_USAGE, - ), - ("bedrock_converse", frozenset({"cache_read_tokens"}) | _CACHE_WRITE_USAGE), - ("gemini_generate", _GEMINI_USAGE), - ("vertex_generate", _GEMINI_USAGE), - ) - } -) class ScriptedToolCall(BaseModel): @@ -166,6 +92,32 @@ class ScriptedUsage(BaseModel): file_search_calls: int = 0 +class WireSpec(BaseModel): + model_config = ConfigDict(frozen=True) + + shape: Shape + mount: str + usage: frozenset[str] + terminals: frozenset[TerminalKind] + + +def _load_wires() -> Mapping[str, WireSpec]: + adapter: Final = TypeAdapter(dict[str, WireSpec]) + loaded: Final = adapter.validate_json((Path(__file__).resolve().with_name("wires.json")).read_bytes()) + known_usage_fields: Final = frozenset(ScriptedUsage.model_fields) - _BASE_USAGE_FIELDS + unknown: Final = { + wire: sorted(spec.usage - known_usage_fields) + for wire, spec in loaded.items() + if spec.usage - known_usage_fields + } + if unknown: + raise ValueError(f"wires.json has unknown usage fields: {unknown}") + return MappingProxyType(loaded) + + +WIRES: Final[Mapping[str, WireSpec]] = _load_wires() + + class ScriptedOutput(BaseModel): model_config = ConfigDict(frozen=True) @@ -205,9 +157,14 @@ class Scenario(BaseModel): @model_validator(mode="after") def _check_terminal_supported(self) -> Scenario: + spec: Final = WIRES.get(self.wire) + if spec is None: + raise ValueError( + f"unknown wire {self.wire}; known wires: {', '.join(sorted(WIRES))}" + ) if ( self.output.terminal != "completed" - and self.output.terminal not in _TERMINAL_CAPS.get(self.wire, frozenset()) + and self.output.terminal not in spec.terminals ): raise ValueError( f"wire {self.wire} cannot emit terminal={self.output.terminal}" @@ -216,7 +173,7 @@ class Scenario(BaseModel): field for field in self.usage.model_fields_set if getattr(self.usage, field) - and field not in (_USAGE_CAPS.get(self.wire, frozenset()) | _BASE_USAGE_FIELDS) + and field not in (spec.usage | _BASE_USAGE_FIELDS) ) if unsupported: raise ValueError( @@ -230,7 +187,7 @@ class Scenario(BaseModel): @property def mount(self) -> str: - return WIRE_MOUNTS[self.wire] + return WIRES[self.wire].mount class ScenarioRegistered(BaseModel): @@ -1266,33 +1223,32 @@ def _render( return RenderedResponse( 200, "application/json", _json_bytes(_responses_body(scenario, requested_model)) ) - if scenario.wire == "bedrock_converse": - if stream: - return RenderedResponse( - 200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario) - ) - return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario))) - if scenario.wire == "vertex_generate": - if stream: - return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) - if scenario.wire == "anthropic_messages": - if stream: - return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model))) - if scenario.wire == "gemini_generate": - if stream: - return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) - if scenario.wire == "openai_responses": - if stream: - return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) - # openai_chat, together_chat, fireworks_chat and azure_chat share the - # OpenAI chat shape. - if stream: - return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) + shape: Final = WIRES[scenario.wire].shape + match shape: + case "bedrock_converse": + if stream: + return RenderedResponse( + 200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario) + ) + return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario))) + case "gemini_generate": + if stream: + return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) + case "anthropic_messages": + if stream: + return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model))) + case "openai_responses": + if stream: + return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) + case "openai_chat": + if stream: + return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) + return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) + case _: + assert_never(shape) # ---------- registry + request routing ---------- diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index b3e6336dcee..c24212c489c 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -19,12 +19,12 @@ from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations from integration._support.scripted_wires import ( - WIRE_MOUNTS, RenderedResponse, Scenario, ScenarioDeleted, ScenarioRegistered, ScenarioStore, + WIRES, Wire, render, ) @@ -218,7 +218,7 @@ class ScenarioHandle: return f"{self.control_url}/{self.scenario_id}/{self._mount()}" def _mount(self) -> str: - return WIRE_MOUNTS[self.wire] + return WIRES[self.wire].mount def register_scenario(scenario: Scenario) -> ScenarioHandle: diff --git a/tests/integration/_support/wires.json b/tests/integration/_support/wires.json new file mode 100644 index 00000000000..b298ccd33aa --- /dev/null +++ b/tests/integration/_support/wires.json @@ -0,0 +1,119 @@ +{ + "openai_chat": { + "shape": "openai_chat", + "mount": "openai", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "web_search_calls" + ], + "terminals": [] + }, + "openai_responses": { + "shape": "openai_responses", + "mount": "openai", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "web_search_calls", + "file_search_calls" + ], + "terminals": [ + "incomplete", + "unvalidated" + ] + }, + "anthropic_messages": { + "shape": "anthropic_messages", + "mount": "anthropic", + "usage": [ + "cache_read_tokens", + "web_search_calls", + "cache_write_5m_tokens", + "cache_write_1h_tokens" + ], + "terminals": [] + }, + "gemini_generate": { + "shape": "gemini_generate", + "mount": "gemini", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "image_input_tokens", + "video_input_tokens", + "web_search_calls", + "google_maps_calls" + ], + "terminals": [ + "prompt_blocked" + ] + }, + "together_chat": { + "shape": "openai_chat", + "mount": "together", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "web_search_calls" + ], + "terminals": [] + }, + "fireworks_chat": { + "shape": "openai_chat", + "mount": "fireworks", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "web_search_calls" + ], + "terminals": [] + }, + "azure_chat": { + "shape": "openai_chat", + "mount": "azure", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "web_search_calls" + ], + "terminals": [] + }, + "bedrock_converse": { + "shape": "bedrock_converse", + "mount": "bedrock", + "usage": [ + "cache_read_tokens", + "cache_write_5m_tokens", + "cache_write_1h_tokens" + ], + "terminals": [] + }, + "vertex_generate": { + "shape": "gemini_generate", + "mount": "vertex", + "usage": [ + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "image_input_tokens", + "video_input_tokens", + "web_search_calls", + "google_maps_calls" + ], + "terminals": [ + "prompt_blocked" + ] + } +} diff --git a/tests/integration/cost_calculation/cases.json b/tests/integration/cost_calculation/cases.json index d2cdd40aa94..8ff6783ae6c 100644 --- a/tests/integration/cost_calculation/cases.json +++ b/tests/integration/cost_calculation/cases.json @@ -1,4 +1,78 @@ { + "providers": [ + { + "litellm_provider": "openai", + "mode": "chat", + "wire": "openai_chat", + "model_prefix": "openai", + "litellm_params": {} + }, + { + "litellm_provider": "openai", + "mode": "responses", + "wire": "openai_responses", + "model_prefix": "openai/responses", + "litellm_params": {} + }, + { + "litellm_provider": "anthropic", + "mode": "chat", + "wire": "anthropic_messages", + "model_prefix": "anthropic", + "litellm_params": {} + }, + { + "litellm_provider": "gemini", + "mode": "chat", + "wire": "gemini_generate", + "model_prefix": null, + "litellm_params": {} + }, + { + "litellm_provider": "together_ai", + "mode": "chat", + "wire": "together_chat", + "model_prefix": null, + "litellm_params": {} + }, + { + "litellm_provider": "fireworks_ai", + "mode": "chat", + "wire": "fireworks_chat", + "model_prefix": null, + "litellm_params": {} + }, + { + "litellm_provider": "azure", + "mode": "chat", + "wire": "azure_chat", + "model_prefix": null, + "litellm_params": { + "api_version": "2025-04-01-preview" + } + }, + { + "litellm_provider": "bedrock_converse", + "mode": "chat", + "wire": "bedrock_converse", + "model_prefix": "bedrock/converse", + "litellm_params": { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1" + } + }, + { + "litellm_provider": "vertex_ai-language-models", + "mode": "chat", + "wire": "vertex_generate", + "model_prefix": "vertex_ai", + "litellm_params": { + "vertex_project": "cc-scripted-project", + "vertex_location": "us-central1" + } + } + ], "deployments": [ { "map_key": "azure/gpt-5.4-mini", diff --git a/tests/integration/cost_calculation/cost_matrix.py b/tests/integration/cost_calculation/cost_matrix.py index 8b9e0aa9424..db054edd321 100644 --- a/tests/integration/cost_calculation/cost_matrix.py +++ b/tests/integration/cost_calculation/cost_matrix.py @@ -27,7 +27,14 @@ from types import MappingProxyType from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, TypeAdapter -from integration._support.scripted_wires import Scenario, ScriptedOutput, ScriptedToolCall, ScriptedUsage, Wire +from integration._support.scripted_wires import ( + WIRES, + Scenario, + ScriptedOutput, + ScriptedToolCall, + ScriptedUsage, + Wire, +) COST_MAP_PATH: Final = Path(__file__).resolve().parent / "cost_map.json" CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" @@ -251,9 +258,20 @@ class Case(BaseModel): ) +class _ProviderWiringRow(BaseModel): + model_config = ConfigDict(frozen=True) + + litellm_provider: str + mode: str + wire: str + model_prefix: str | None + litellm_params: Mapping[str, str] + + class _CasesFile(BaseModel): model_config = ConfigDict(frozen=True) + providers: tuple[_ProviderWiringRow, ...] = () deployments: tuple[DeploymentSpec, ...] = () cases: tuple[Case, ...] = () @@ -267,7 +285,7 @@ _DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( @dataclass(frozen=True, slots=True) class _ProviderWiring: - """How a (litellm_provider, mode) pair maps to a sidecar wire, the provider + """How a (litellm_provider, mode) pair maps to a provider wire, the provider prefix on the registered litellm model string, and extra litellm_params.""" wire: Wire @@ -275,42 +293,26 @@ class _ProviderWiring: litellm_params: Mapping[str, str] -_AZURE_PARAMS: Final[Mapping[str, str]] = MappingProxyType({"api_version": "2025-04-01-preview"}) -_BEDROCK_PARAMS: Final[Mapping[str, str]] = MappingProxyType( - { - "aws_access_key_id": "AKIASCRIPTEDPROVIDER", - "aws_secret_access_key": "scripted-secret", - "aws_region_name": "us-east-1", - } -) -_VERTEX_PARAMS: Final[Mapping[str, str]] = MappingProxyType( - { - "vertex_project": "cc-scripted-project", - "vertex_location": "us-central1", - } -) +def _provider_wiring(rows: tuple[_ProviderWiringRow, ...]) -> Mapping[tuple[str, str], _ProviderWiring]: + unknown_wires: Final = sorted({row.wire for row in rows if row.wire not in WIRES}) + if unknown_wires: + raise ValueError( + f"cases.json providers has unknown wires: {unknown_wires}; " + f"known wires are {sorted(WIRES)}" + ) + return MappingProxyType( + { + (row.litellm_provider, row.mode): _ProviderWiring( + row.wire, + row.model_prefix, + MappingProxyType(dict(row.litellm_params)), + ) + for row in rows + } + ) -_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = MappingProxyType( - { - ("openai", "chat"): _ProviderWiring("openai_chat", "openai", MappingProxyType({})), - ("openai", "responses"): _ProviderWiring( - "openai_responses", "openai/responses", MappingProxyType({}) - ), - ("anthropic", "chat"): _ProviderWiring( - "anthropic_messages", "anthropic", MappingProxyType({}) - ), - ("gemini", "chat"): _ProviderWiring("gemini_generate", None, MappingProxyType({})), - ("together_ai", "chat"): _ProviderWiring("together_chat", None, MappingProxyType({})), - ("fireworks_ai", "chat"): _ProviderWiring("fireworks_chat", None, MappingProxyType({})), - ("azure", "chat"): _ProviderWiring("azure_chat", None, _AZURE_PARAMS), - ("bedrock_converse", "chat"): _ProviderWiring( - "bedrock_converse", "bedrock/converse", _BEDROCK_PARAMS - ), - ("vertex_ai-language-models", "chat"): _ProviderWiring( - "vertex_generate", "vertex_ai", _VERTEX_PARAMS - ), - } -) + +_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = _provider_wiring(CASES_FILE.providers) @dataclass(frozen=True, slots=True) @@ -390,11 +392,7 @@ def _frontier() -> tuple[FrontierModel, ...]: pair = (entry.litellm_provider, entry.mode) wiring = _PROVIDER_WIRING.get(pair) if wiring is None: - raise ValueError( - f"cost_map entry {map_key} has no wiring for " - f"(litellm_provider={pair[0]}, mode={pair[1]}); add a " - f"_ProviderWiring row in cost_matrix.py" - ) + continue siblings = groups[pair] override_key = ( siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None @@ -567,6 +565,13 @@ def matrix_data_errors() -> tuple[str, ...]: for case in CASES if (case.family == "transport") != (not case.owns and not case.fallback_for) ) + missing_provider_rows: Final = sorted( + f"cost_map entry {map_key} has no providers row for " + f"(litellm_provider={entry.litellm_provider}, mode={entry.mode}); " + f"add a providers row in cases.json" + for map_key, entry in COST_MAP.items() + if (entry.litellm_provider, entry.mode) not in _PROVIDER_WIRING + ) input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) findings: Final = ( ( @@ -615,5 +620,10 @@ def matrix_data_errors() -> tuple[str, ...]: if family_violations else None ), + ( + f"cost_map entries without providers rows: {missing_provider_rows}" + if missing_provider_rows + else None + ), ) return tuple(finding for finding in findings if finding is not None) diff --git a/tests/integration/cost_calculation/test_token_pricing.py b/tests/integration/cost_calculation/test_token_pricing.py index 69e2ac7ca0c..0b4e9948dfa 100644 --- a/tests/integration/cost_calculation/test_token_pricing.py +++ b/tests/integration/cost_calculation/test_token_pricing.py @@ -9,7 +9,7 @@ import pytest from pydantic import JsonValue from integration._support.client import JSON_OBJECT, Gateway -from integration._support.scripted_wires import ScriptedUsage, Wire +from integration._support.scripted_wires import WIRES, ScriptedUsage, Wire from integration.cost_calculation.conftest import ( approx_equal, assert_total_is_sum_of_components, @@ -50,12 +50,12 @@ _MATRIX: Final = tuple( for model in FRONTIER_MODELS for case in cases_for(model) ) -_CACHE_WIRES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) -_WEB_SEARCH_OPTION_WIRES: Final = frozenset({"openai_chat", "azure_chat", "openai_responses"}) +_CACHE_SHAPES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) +_WEB_SEARCH_OPTION_SHAPES: Final = frozenset({"openai_chat", "openai_responses"}) def _cache_control(usage: ScriptedUsage, wire: Wire) -> dict[str, JsonValue] | None: - if wire not in _CACHE_WIRES: + if WIRES[wire].shape not in _CACHE_SHAPES: return None if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens): return None @@ -148,7 +148,7 @@ def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) - **({"audio": {"voice": "alloy", "format": "pcm16"}} if case.audio_output else {}), **( {"web_search_options": {"search_context_size": case.web_search}} - if case.web_search is not None and model.wire in _WEB_SEARCH_OPTION_WIRES + if case.web_search is not None and WIRES[model.wire].shape in _WEB_SEARCH_OPTION_SHAPES else {} ), **({"tools": tools} if tools else {}), From 380ec1a004e71b518bd417bd3023df570b2ee454 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 01:43:00 +0000 Subject: [PATCH 28/30] docs(integration): keep cost map loading note in README Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/README.md b/tests/integration/README.md index a007eb6dc68..7e3cf67cb08 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The `cost` group runs the scripted-wire cost matrix through the shared integration upstream. A provider speaking an existing response shape is a `wires.json` row, a `cases.json` `providers` row and cost-map entries; a new response shape needs a renderer in `scripted_wires.py` +The `cost` group runs the scripted-wire cost matrix through the shared integration upstream, which serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry. A provider speaking an existing response shape is a `wires.json` row, a `cases.json` `providers` row and cost-map entries; a new response shape needs a renderer in `scripted_wires.py` Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate From 57d2fefa8dd62ab596fa9a77677fc420a4ddfa68 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 03:16:16 +0000 Subject: [PATCH 29/30] test(integration): derive scripted shapes from litellm provider configs Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 2 +- .../{scripted_wires.py => scripted_shapes.py} | 198 ++++++++++-------- tests/integration/_support/upstream.py | 11 +- tests/integration/_support/wires.json | 119 ----------- tests/integration/cost_calculation/cases.json | 9 - .../integration/cost_calculation/conftest.py | 2 +- .../cost_calculation/cost_matrix.py | 113 ++++++---- .../cost_calculation/test_token_pricing.py | 24 +-- 8 files changed, 196 insertions(+), 282 deletions(-) rename tests/integration/_support/{scripted_wires.py => scripted_shapes.py} (91%) delete mode 100644 tests/integration/_support/wires.json diff --git a/tests/integration/README.md b/tests/integration/README.md index 7e3cf67cb08..dcdf0e9fa96 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The `cost` group runs the scripted-wire cost matrix through the shared integration upstream, which serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry. A provider speaking an existing response shape is a `wires.json` row, a `cases.json` `providers` row and cost-map entries; a new response shape needs a renderer in `scripted_wires.py` +The `cost` group runs the scripted-shape cost matrix through the shared integration upstream, which serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry. The upstream renders a scenario in the shape LiteLLM's own provider config resolves to for the deployment, so a provider LiteLLM already parses with one of the five rendered families is a cost-map entry plus a `cases.json` `providers` row with its deployment parameters; a provider whose config class is none of those families fails at collection until `scripted_shapes.py` gains a renderer Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate diff --git a/tests/integration/_support/scripted_wires.py b/tests/integration/_support/scripted_shapes.py similarity index 91% rename from tests/integration/_support/scripted_wires.py rename to tests/integration/_support/scripted_shapes.py index 8da2c57c9a0..61bfe7c24f1 100644 --- a/tests/integration/_support/scripted_wires.py +++ b/tests/integration/_support/scripted_shapes.py @@ -1,24 +1,20 @@ -"""Scripted provider wires for the cost-calculation integration suite. +"""Scripted response shapes for the cost-calculation integration suite. -The shared integration upstream registers a Scenario over a small control API; -the provider wire routes answer the proxy's upstream calls with the scripted usage figures, in the exact wire shape -the real provider would emit (OpenAI chat completions, OpenAI Responses, -Anthropic Messages, Gemini generateContent, or the OpenAI-compatible Together / -Fireworks surfaces). Because the usage is scripted, expected spend is literal -arithmetic on the test cost map's rates, with no dependency on what a real -provider would report. +This module owns the Scenario schema, the five renderers, one per LiteLLM +parser family, and the dispatcher. Because the usage is scripted, expected +spend is literal arithmetic on the test cost map's rates, with no dependency +on what a real provider would report. The upstream exposes: - ``POST /__scenarios`` register a Scenario JSON, returns its id - ``DELETE /__scenarios/`` remove it -- ``POST ///`` provider wire; mount is one of - ``openai``, ``anthropic``, ``gemini``, ``together``, ``fireworks``, ``azure``, - ``bedrock``, ``vertex`` and the remainder is whatever path the provider - client appends (``chat/completions``, ``responses``, ``v1/messages``, - ``models/:generateContent`` ...). Vertex appends ``:generateContent`` / - ``:streamGenerateContent`` to the mount segment itself, and Bedrock Converse - targets ``model//converse`` / ``converse-stream`` +- ``POST //`` provider response; the remainder is whatever + path the provider client appends (``chat/completions``, ``responses``, + ``v1/messages``, ``models/:generateContent`` ...). Vertex appends + ``:generateContent`` / ``:streamGenerateContent`` to the scenario segment, + and Bedrock Converse targets ``model//converse`` / + ``converse-stream`` A request carrying ``"stream": true`` (or the ``:streamGenerateContent`` Gemini verb) gets an SSE answer; ``stream_usage`` on the Scenario decides whether the @@ -34,14 +30,12 @@ import time import zlib from collections.abc import Mapping from dataclasses import dataclass -from pathlib import Path from types import MappingProxyType from typing import Final, Literal, TypeAlias, assert_never from urllib.parse import unquote, urlsplit from pydantic import BaseModel, ConfigDict, TypeAdapter, model_validator -Wire: TypeAlias = str Shape: TypeAlias = Literal[ "openai_chat", "openai_responses", @@ -49,6 +43,77 @@ Shape: TypeAlias = Literal[ "gemini_generate", "bedrock_converse", ] + + +@dataclass(frozen=True, slots=True) +class ShapeSpec: + usage: frozenset[str] + terminals: frozenset[str] + + +SHAPES: Final[Mapping[Shape, ShapeSpec]] = MappingProxyType( + { + "openai_chat": ShapeSpec( + usage=frozenset( + { + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "web_search_calls", + } + ), + terminals=frozenset(), + ), + "openai_responses": ShapeSpec( + usage=frozenset( + { + "cache_read_tokens", + "reasoning_tokens", + "web_search_calls", + "file_search_calls", + } + ), + terminals=frozenset({"incomplete", "unvalidated"}), + ), + "anthropic_messages": ShapeSpec( + usage=frozenset( + { + "cache_read_tokens", + "web_search_calls", + "cache_write_5m_tokens", + "cache_write_1h_tokens", + } + ), + terminals=frozenset(), + ), + "gemini_generate": ShapeSpec( + usage=frozenset( + { + "cache_read_tokens", + "reasoning_tokens", + "audio_input_tokens", + "audio_output_tokens", + "image_input_tokens", + "video_input_tokens", + "web_search_calls", + "google_maps_calls", + } + ), + terminals=frozenset({"prompt_blocked"}), + ), + "bedrock_converse": ShapeSpec( + usage=frozenset( + { + "cache_read_tokens", + "cache_write_5m_tokens", + "cache_write_1h_tokens", + } + ), + terminals=frozenset(), + ), + } +) StreamUsage: TypeAlias = Literal["final_chunk", "absent"] ServiceTier: TypeAlias = Literal["flex", "priority"] TerminalKind: TypeAlias = Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] @@ -58,7 +123,7 @@ _BASE_USAGE_FIELDS: Final = frozenset({"fresh_input_tokens", "output_tokens"}) class ScriptedToolCall(BaseModel): """A single function call the scripted output emits instead of text. - ``arguments`` is the wire's JSON string (~250 chars), sliced into deltas + ``arguments`` is the shape's JSON string (~250 chars), sliced into deltas for streams.""" model_config = ConfigDict(frozen=True) @@ -71,7 +136,7 @@ class ScriptedUsage(BaseModel): """Physical token counts the scripted response reports. ``fresh_input_tokens`` is the uncached, never-written, non-audio input count; ``output_tokens`` is the non-reasoning, non-audio output count. Renderers add the cached, written, - audio, and reasoning counts into the wire's total fields the way the real + audio, and reasoning counts into the shape's total fields the way the real provider does (inside prompt_tokens for OpenAI/Gemini, as uncached-only input_tokens for Anthropic).""" @@ -92,32 +157,6 @@ class ScriptedUsage(BaseModel): file_search_calls: int = 0 -class WireSpec(BaseModel): - model_config = ConfigDict(frozen=True) - - shape: Shape - mount: str - usage: frozenset[str] - terminals: frozenset[TerminalKind] - - -def _load_wires() -> Mapping[str, WireSpec]: - adapter: Final = TypeAdapter(dict[str, WireSpec]) - loaded: Final = adapter.validate_json((Path(__file__).resolve().with_name("wires.json")).read_bytes()) - known_usage_fields: Final = frozenset(ScriptedUsage.model_fields) - _BASE_USAGE_FIELDS - unknown: Final = { - wire: sorted(spec.usage - known_usage_fields) - for wire, spec in loaded.items() - if spec.usage - known_usage_fields - } - if unknown: - raise ValueError(f"wires.json has unknown usage fields: {unknown}") - return MappingProxyType(loaded) - - -WIRES: Final[Mapping[str, WireSpec]] = _load_wires() - - class ScriptedOutput(BaseModel): model_config = ConfigDict(frozen=True) @@ -127,9 +166,9 @@ class ScriptedOutput(BaseModel): # prove the biller prices the provider-reported model. response_model: str | None = None # OpenAI-compatible providers can report a provider-computed cost; emitted as - # the top-level "cost" field on the together/fireworks wire. + # the top-level "cost" field on the together/fireworks response. provider_cost: float | None = None - # When set, the response is a tool call only: no text content on any wire. + # When set, the response is a tool call only: no text content on any response. tool_call: ScriptedToolCall | None = None # Terminal shape: "unvalidated" makes the Responses terminal response fail # pydantic validation so the proxy takes its model_construct dict path; @@ -141,7 +180,7 @@ class Scenario(BaseModel): model_config = ConfigDict(frozen=True) scenario_id: str - wire: Wire + shape: Shape usage: ScriptedUsage output: ScriptedOutput # The bare provider-facing model name the renderer echoes when the request @@ -157,17 +196,13 @@ class Scenario(BaseModel): @model_validator(mode="after") def _check_terminal_supported(self) -> Scenario: - spec: Final = WIRES.get(self.wire) - if spec is None: - raise ValueError( - f"unknown wire {self.wire}; known wires: {', '.join(sorted(WIRES))}" - ) + spec: Final = SHAPES[self.shape] if ( self.output.terminal != "completed" and self.output.terminal not in spec.terminals ): raise ValueError( - f"wire {self.wire} cannot emit terminal={self.output.terminal}" + f"shape {self.shape} cannot emit terminal={self.output.terminal}" ) unsupported: Final = frozenset( field @@ -177,18 +212,14 @@ class Scenario(BaseModel): ) if unsupported: raise ValueError( - f"wire {self.wire} cannot express usage fields {sorted(unsupported)}" + f"shape {self.shape} cannot express usage fields {sorted(unsupported)}" ) - if (self.speed or self.inference_geo) and self.wire != "anthropic_messages": + if (self.speed or self.inference_geo) and self.shape != "anthropic_messages": raise ValueError( - f"wire {self.wire} cannot emit speed/inference_geo (anthropic usage fields)" + f"shape {self.shape} cannot emit speed/inference_geo (anthropic usage fields)" ) return self - @property - def mount(self) -> str: - return WIRES[self.wire].mount - class ScenarioRegistered(BaseModel): scenario_id: str @@ -233,7 +264,7 @@ def _sse(events: tuple[tuple[str | None, Mapping[str, object] | str], ...]) -> b return "".join(_sse_frame(event_name, data) for event_name, data in events).encode("utf-8") -# ---------- per-wire usage shapes ---------- + # ---------- per-shape usage shapes ---------- def _openai_usage(u: ScriptedUsage) -> Mapping[str, object]: @@ -400,7 +431,7 @@ def _responses_usage(u: ScriptedUsage) -> Mapping[str, object]: ) -# ---------- per-wire responses ---------- + # ---------- per-shape responses ---------- def _split_arguments(arguments: str) -> tuple[str, ...]: @@ -1214,8 +1245,8 @@ def _render( scenario: Scenario, *, stream: bool, requested_model: str, path_tail: str ) -> RenderedResponse: # Azure bridges gpt-5.4+ chat requests carrying function tools onto the - # Responses API, which lands on the same mount at openai/responses. - if scenario.wire == "azure_chat" and path_tail.endswith("openai/responses"): + # Responses API, which lands on the same shape at openai/responses. + if scenario.shape == "openai_chat" and path_tail.endswith("openai/responses"): if stream: return RenderedResponse( 200, "text/event-stream", _responses_sse(scenario, requested_model) @@ -1223,7 +1254,7 @@ def _render( return RenderedResponse( 200, "application/json", _json_bytes(_responses_body(scenario, requested_model)) ) - shape: Final = WIRES[scenario.wire].shape + shape: Final = scenario.shape match shape: case "bedrock_converse": if stream: @@ -1282,8 +1313,8 @@ def _request_body(body: bytes) -> Mapping[str, object]: return MappingProxyType({}) -def _request_wants_stream(mount_endpoint: str | None, path_tail: str, body: bytes) -> bool: - if mount_endpoint == "streamGenerateContent" or ":streamGenerateContent" in path_tail: +def _request_wants_stream(endpoint: str | None, path_tail: str, body: bytes) -> bool: + if endpoint == "streamGenerateContent" or ":streamGenerateContent" in path_tail: return True if path_tail.endswith("converse-stream"): return True @@ -1301,44 +1332,33 @@ def _request_model(body: bytes, path_tail: str, scenario: Scenario) -> str: path_model: Final = path_tail.split("/", 2)[1] if path_tail.count("/") >= 2 else "" if path_model: return unquote(path_model) - # Vertex names it in the URL too, but the mount segment swallowed it when - # the api_base carried a path; fall back to the scenario's declared model. + # Vertex names it in the URL too, but the path may carry only the endpoint; + # fall back to the scenario's declared model. return scenario.model def render(store: ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: path: Final = urlsplit(raw_path).path segments: Final = tuple(segment for segment in path.split("/") if segment) - if len(segments) < 2 or method != "POST": + if len(segments) < 1 or method != "POST": return RenderedResponse( 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) ) - scenario_id: Final = segments[0] - # Vertex builds {api_base}:{endpoint}, so the mount segment can carry a - # :generateContent / :streamGenerateContent suffix. - mount_segment: Final = segments[1] - mount, mount_endpoint = ( - mount_segment.split(":", 1) - if ":" in mount_segment - else (mount_segment, None) + scenario_segment: Final = segments[0] + scenario_id, endpoint = ( + scenario_segment.split(":", 1) + if ":" in scenario_segment + else (scenario_segment, None) ) found: Final = store.get(scenario_id) if found is None: return RenderedResponse( 404, "application/json", _json_bytes(_jobj(("error", f"unknown scenario {scenario_id}"))) ) - if found.mount != mount: - return RenderedResponse( - 400, - "application/json", - _json_bytes( - _jobj(("error", f"scenario {scenario_id} is wire {found.wire}, not mount {mount}")) - ), - ) - tail: Final = "/".join(segments[2:]) + tail: Final = "/".join(segments[1:]) return _render( found, - stream=_request_wants_stream(mount_endpoint, tail, body), + stream=_request_wants_stream(endpoint, tail, body), requested_model=_request_model(body, tail, found), path_tail=tail, ) diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index c24212c489c..5374d420b6a 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -18,14 +18,12 @@ from starlette.responses import JSONResponse, Response from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations -from integration._support.scripted_wires import ( +from integration._support.scripted_shapes import ( RenderedResponse, Scenario, ScenarioDeleted, ScenarioRegistered, ScenarioStore, - WIRES, - Wire, render, ) @@ -211,14 +209,10 @@ CONTROL_URL: Final = os.environ.get("INTEGRATION_UPSTREAM_URL", "http://127.0.0. @dataclass(frozen=True, slots=True) class ScenarioHandle: scenario_id: str - wire: Wire control_url: str def api_base(self) -> str: - return f"{self.control_url}/{self.scenario_id}/{self._mount()}" - - def _mount(self) -> str: - return WIRES[self.wire].mount + return f"{self.control_url}/{self.scenario_id}" def register_scenario(scenario: Scenario) -> ScenarioHandle: @@ -232,7 +226,6 @@ def register_scenario(scenario: Scenario) -> ScenarioHandle: result: Final = ScenarioRegistered.model_validate_json(response.content) return ScenarioHandle( scenario_id=result.scenario_id, - wire=scenario.wire, control_url=CONTROL_URL, ) diff --git a/tests/integration/_support/wires.json b/tests/integration/_support/wires.json deleted file mode 100644 index b298ccd33aa..00000000000 --- a/tests/integration/_support/wires.json +++ /dev/null @@ -1,119 +0,0 @@ -{ - "openai_chat": { - "shape": "openai_chat", - "mount": "openai", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "web_search_calls" - ], - "terminals": [] - }, - "openai_responses": { - "shape": "openai_responses", - "mount": "openai", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "web_search_calls", - "file_search_calls" - ], - "terminals": [ - "incomplete", - "unvalidated" - ] - }, - "anthropic_messages": { - "shape": "anthropic_messages", - "mount": "anthropic", - "usage": [ - "cache_read_tokens", - "web_search_calls", - "cache_write_5m_tokens", - "cache_write_1h_tokens" - ], - "terminals": [] - }, - "gemini_generate": { - "shape": "gemini_generate", - "mount": "gemini", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "image_input_tokens", - "video_input_tokens", - "web_search_calls", - "google_maps_calls" - ], - "terminals": [ - "prompt_blocked" - ] - }, - "together_chat": { - "shape": "openai_chat", - "mount": "together", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "web_search_calls" - ], - "terminals": [] - }, - "fireworks_chat": { - "shape": "openai_chat", - "mount": "fireworks", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "web_search_calls" - ], - "terminals": [] - }, - "azure_chat": { - "shape": "openai_chat", - "mount": "azure", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "web_search_calls" - ], - "terminals": [] - }, - "bedrock_converse": { - "shape": "bedrock_converse", - "mount": "bedrock", - "usage": [ - "cache_read_tokens", - "cache_write_5m_tokens", - "cache_write_1h_tokens" - ], - "terminals": [] - }, - "vertex_generate": { - "shape": "gemini_generate", - "mount": "vertex", - "usage": [ - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "image_input_tokens", - "video_input_tokens", - "web_search_calls", - "google_maps_calls" - ], - "terminals": [ - "prompt_blocked" - ] - } -} diff --git a/tests/integration/cost_calculation/cases.json b/tests/integration/cost_calculation/cases.json index 8ff6783ae6c..478aa069f1e 100644 --- a/tests/integration/cost_calculation/cases.json +++ b/tests/integration/cost_calculation/cases.json @@ -3,49 +3,42 @@ { "litellm_provider": "openai", "mode": "chat", - "wire": "openai_chat", "model_prefix": "openai", "litellm_params": {} }, { "litellm_provider": "openai", "mode": "responses", - "wire": "openai_responses", "model_prefix": "openai/responses", "litellm_params": {} }, { "litellm_provider": "anthropic", "mode": "chat", - "wire": "anthropic_messages", "model_prefix": "anthropic", "litellm_params": {} }, { "litellm_provider": "gemini", "mode": "chat", - "wire": "gemini_generate", "model_prefix": null, "litellm_params": {} }, { "litellm_provider": "together_ai", "mode": "chat", - "wire": "together_chat", "model_prefix": null, "litellm_params": {} }, { "litellm_provider": "fireworks_ai", "mode": "chat", - "wire": "fireworks_chat", "model_prefix": null, "litellm_params": {} }, { "litellm_provider": "azure", "mode": "chat", - "wire": "azure_chat", "model_prefix": null, "litellm_params": { "api_version": "2025-04-01-preview" @@ -54,7 +47,6 @@ { "litellm_provider": "bedrock_converse", "mode": "chat", - "wire": "bedrock_converse", "model_prefix": "bedrock/converse", "litellm_params": { "aws_access_key_id": "AKIASCRIPTEDPROVIDER", @@ -65,7 +57,6 @@ { "litellm_provider": "vertex_ai-language-models", "mode": "chat", - "wire": "vertex_generate", "model_prefix": "vertex_ai", "litellm_params": { "vertex_project": "cc-scripted-project", diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 0cbc837c184..9229bb47817 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -136,7 +136,7 @@ def register_scenario_deployment( **model.litellm_params, **( {"vertex_credentials": _vertex_service_account_json(control_url)} - if model.wire == "vertex_generate" + if model.llm_provider == "vertex_ai" else {} ), } diff --git a/tests/integration/cost_calculation/cost_matrix.py b/tests/integration/cost_calculation/cost_matrix.py index db054edd321..b261deb68b2 100644 --- a/tests/integration/cost_calculation/cost_matrix.py +++ b/tests/integration/cost_calculation/cost_matrix.py @@ -26,14 +26,22 @@ from pathlib import Path from types import MappingProxyType from typing import Final, Literal +from litellm import get_llm_provider +from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig +from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager from pydantic import BaseModel, ConfigDict, Field, TypeAdapter -from integration._support.scripted_wires import ( - WIRES, +from integration._support.scripted_shapes import ( Scenario, + Shape, ScriptedOutput, ScriptedToolCall, ScriptedUsage, - Wire, ) COST_MAP_PATH: Final = Path(__file__).resolve().parent / "cost_map.json" @@ -151,8 +159,8 @@ def _entry_has_rate_key(entry: CostMapEntry, rate_key: str) -> bool: return value is not None -SERVICE_TIER_REQUEST_WIRES: Final = frozenset( - {"openai_chat", "azure_chat", "openai_responses", "bedrock_converse"} +SERVICE_TIER_REQUEST_SHAPES: Final = frozenset( + {"openai_chat", "openai_responses", "bedrock_converse"} ) @@ -240,7 +248,7 @@ class Case(BaseModel): def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: return Scenario( scenario_id=scenario_id, - wire=model.wire, + shape=model.shape, usage=self.usage_for(model.map_key), model=model.provider_model, output=ScriptedOutput( @@ -263,7 +271,6 @@ class _ProviderWiringRow(BaseModel): litellm_provider: str mode: str - wire: str model_prefix: str | None litellm_params: Mapping[str, str] @@ -284,26 +291,19 @@ _DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( @dataclass(frozen=True, slots=True) -class _ProviderWiring: - """How a (litellm_provider, mode) pair maps to a provider wire, the provider - prefix on the registered litellm model string, and extra litellm_params.""" +class _DeploymentDefaults: + """How a (litellm_provider, mode) pair maps to deployment defaults.""" - wire: Wire model_prefix: str | None litellm_params: Mapping[str, str] -def _provider_wiring(rows: tuple[_ProviderWiringRow, ...]) -> Mapping[tuple[str, str], _ProviderWiring]: - unknown_wires: Final = sorted({row.wire for row in rows if row.wire not in WIRES}) - if unknown_wires: - raise ValueError( - f"cases.json providers has unknown wires: {unknown_wires}; " - f"known wires are {sorted(WIRES)}" - ) +def _deployment_defaults( + rows: tuple[_ProviderWiringRow, ...], +) -> Mapping[tuple[str, str], _DeploymentDefaults]: return MappingProxyType( { - (row.litellm_provider, row.mode): _ProviderWiring( - row.wire, + (row.litellm_provider, row.mode): _DeploymentDefaults( row.model_prefix, MappingProxyType(dict(row.litellm_params)), ) @@ -312,19 +312,22 @@ def _provider_wiring(rows: tuple[_ProviderWiringRow, ...]) -> Mapping[tuple[str, ) -_PROVIDER_WIRING: Final[Mapping[tuple[str, str], _ProviderWiring]] = _provider_wiring(CASES_FILE.providers) +_DEPLOYMENT_DEFAULTS: Final[Mapping[tuple[str, str], _DeploymentDefaults]] = _deployment_defaults( + CASES_FILE.providers +) @dataclass(frozen=True, slots=True) class FrontierModel: """One deployment under test, derived from a cost-map entry: the model_name - the suite registers, the provider-prefixed litellm model string, the wire - the scripted upstream speaks, and the sibling map model the response_model - override case reports.""" + the suite registers, the provider-prefixed litellm model string, the + response shape the scripted upstream speaks, and the sibling map model the + response_model override case reports.""" model_name: str litellm_model: str - wire: Wire + shape: Shape + llm_provider: str map_key: str override_model: str | None = None override_map_key: str | None = None @@ -343,7 +346,7 @@ class FrontierModel: # override can never repoint pricing there, same as a base_model pin. if ( self.base_model is not None - or self.wire == "bedrock_converse" + or self.shape == "bedrock_converse" or self.override_map_key is None ): return self.rates @@ -371,12 +374,35 @@ def _provider_model(litellm_model: str) -> str: return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) -def _litellm_model_for(map_key: str, wiring: _ProviderWiring) -> str: - if wiring.model_prefix is None: +def _litellm_model_for(map_key: str, defaults: _DeploymentDefaults) -> str: + if defaults.model_prefix is None: return map_key - if map_key.startswith(f"{wiring.model_prefix}/"): + if map_key.startswith(f"{defaults.model_prefix}/"): return map_key - return f"{wiring.model_prefix}/{map_key}" + return f"{defaults.model_prefix}/{map_key}" + + +def _resolve(litellm_model: str, mode: str) -> tuple[str, Shape]: + model, provider, _, _ = get_llm_provider(model=litellm_model) + llm_provider: Final = LlmProviders(provider) + if mode == "responses": + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( + model=model, + provider=llm_provider, + ) + if isinstance(responses_config, OpenAIResponsesAPIConfig): + return provider, "openai_responses" + raise ValueError(f"no scripted renderer for {type(responses_config).__name__} ({litellm_model})") + config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) + if isinstance(config, AmazonConverseConfig): + return provider, "bedrock_converse" + if isinstance(config, VertexGeminiConfig): + return provider, "gemini_generate" + if isinstance(config, AnthropicConfig): + return provider, "anthropic_messages" + if isinstance(config, (AzureOpenAIConfig, OpenAIGPTConfig)): + return provider, "openai_chat" + raise ValueError(f"no scripted renderer for {type(config).__name__} ({litellm_model})") def _frontier() -> tuple[FrontierModel, ...]: @@ -390,26 +416,29 @@ def _frontier() -> tuple[FrontierModel, ...]: for map_key in sorted(COST_MAP): entry = COST_MAP[map_key] pair = (entry.litellm_provider, entry.mode) - wiring = _PROVIDER_WIRING.get(pair) - if wiring is None: + defaults = _DEPLOYMENT_DEFAULTS.get(pair) + if defaults is None: continue siblings = groups[pair] override_key = ( siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None ) override_litellm = ( - _litellm_model_for(override_key, wiring) if override_key is not None else None + _litellm_model_for(override_key, defaults) if override_key is not None else None ) deployment = _DEPLOYMENTS.get(map_key) + litellm_model = ( + deployment.litellm_model + if deployment is not None and deployment.litellm_model is not None + else _litellm_model_for(map_key, defaults) + ) + llm_provider, shape = _resolve(litellm_model, entry.mode) models.append( FrontierModel( model_name=f"cc-{map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", - litellm_model=( - deployment.litellm_model - if deployment is not None and deployment.litellm_model is not None - else _litellm_model_for(map_key, wiring) - ), - wire=wiring.wire, + litellm_model=litellm_model, + shape=shape, + llm_provider=llm_provider, map_key=map_key, override_model=( _provider_model(override_litellm) @@ -418,7 +447,7 @@ def _frontier() -> tuple[FrontierModel, ...]: ), override_map_key=override_key, base_model=deployment.base_model if deployment is not None else None, - litellm_params=wiring.litellm_params, + litellm_params=defaults.litellm_params, ) ) return tuple(models) @@ -471,7 +500,7 @@ def audio_input_data_url() -> str: def video_input_data_url() -> str: """A deterministic mp4-looking blob (ftyp box plus a fixed mdat payload) - as a data URL; only the media type and bytes matter to the wire.""" + as a data URL; only the media type and bytes matter to the response.""" ftyp: Final = struct.pack(">I4s4sI4s4s", 24, b"ftyp", b"isom", 0x200, b"isom", b"iso6") mdat_payload: Final = bytes((i * 7 + 13) % 256 for i in range(4096)) mdat: Final = struct.pack(">I4s", 8 + len(mdat_payload), b"mdat") + mdat_payload @@ -570,7 +599,7 @@ def matrix_data_errors() -> tuple[str, ...]: f"(litellm_provider={entry.litellm_provider}, mode={entry.mode}); " f"add a providers row in cases.json" for map_key, entry in COST_MAP.items() - if (entry.litellm_provider, entry.mode) not in _PROVIDER_WIRING + if (entry.litellm_provider, entry.mode) not in _DEPLOYMENT_DEFAULTS ) input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) findings: Final = ( diff --git a/tests/integration/cost_calculation/test_token_pricing.py b/tests/integration/cost_calculation/test_token_pricing.py index 0b4e9948dfa..cc48da2b819 100644 --- a/tests/integration/cost_calculation/test_token_pricing.py +++ b/tests/integration/cost_calculation/test_token_pricing.py @@ -1,4 +1,4 @@ -"""Token pricing coverage for the integration scripted-wire cost shard.""" +"""Token pricing coverage for the integration scripted-shape cost shard.""" from __future__ import annotations @@ -9,7 +9,7 @@ import pytest from pydantic import JsonValue from integration._support.client import JSON_OBJECT, Gateway -from integration._support.scripted_wires import WIRES, ScriptedUsage, Wire +from integration._support.scripted_shapes import ScriptedUsage, Shape from integration.cost_calculation.conftest import ( approx_equal, assert_total_is_sum_of_components, @@ -20,7 +20,7 @@ from integration.cost_calculation.cost_matrix import ( AUDIO_INPUT_DATA_URL, FRONTIER_MODELS, IMAGE_INPUT_DATA_URL, - SERVICE_TIER_REQUEST_WIRES, + SERVICE_TIER_REQUEST_SHAPES, VIDEO_INPUT_DATA_URL, Case, FrontierModel, @@ -54,8 +54,8 @@ _CACHE_SHAPES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) _WEB_SEARCH_OPTION_SHAPES: Final = frozenset({"openai_chat", "openai_responses"}) -def _cache_control(usage: ScriptedUsage, wire: Wire) -> dict[str, JsonValue] | None: - if WIRES[wire].shape not in _CACHE_SHAPES: +def _cache_control(usage: ScriptedUsage, shape: Shape) -> dict[str, JsonValue] | None: + if shape not in _CACHE_SHAPES: return None if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens): return None @@ -107,18 +107,18 @@ def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) - ), *( [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] - if case.web_search is not None and model.wire == "anthropic_messages" + if case.web_search is not None and model.shape == "anthropic_messages" else [] ), *( [{"googleSearch": {}}] - if case.web_search is not None and model.wire in ("gemini_generate", "vertex_generate") + if case.web_search is not None and model.shape == "gemini_generate" else [] ), *([{"googleMaps": {}}] if case.google_maps else []), *([{"type": "file_search", "vector_store_ids": ["vs_cost_calc_fixture"]}] if case.file_search else []), ] - cache_control: Final = _cache_control(usage, model.wire) + cache_control: Final = _cache_control(usage, model.shape) message: Final = { "role": "system", "content": [ @@ -136,7 +136,7 @@ def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) - **({"stream_options": {"include_usage": True}} if case.stream else {}), **( {"service_tier": case.service_tier} - if case.service_tier is not None and model.wire in SERVICE_TIER_REQUEST_WIRES + if case.service_tier is not None and model.shape in SERVICE_TIER_REQUEST_SHAPES else {} ), **({"reasoning_effort": "medium"} if case.reasoning else {}), @@ -148,15 +148,15 @@ def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) - **({"audio": {"voice": "alloy", "format": "pcm16"}} if case.audio_output else {}), **( {"web_search_options": {"search_context_size": case.web_search}} - if case.web_search is not None and WIRES[model.wire].shape in _WEB_SEARCH_OPTION_SHAPES + if case.web_search is not None and model.shape in _WEB_SEARCH_OPTION_SHAPES else {} ), **({"tools": tools} if tools else {}), - **({"tool_choice": "auto"} if case.tool_call and model.wire != "bedrock_converse" else {}), + **({"tool_choice": "auto"} if case.tool_call and model.shape != "bedrock_converse" else {}), "allowed_openai_params": [ name for name, sent in ( - ("tool_choice", case.tool_call and model.wire != "bedrock_converse"), + ("tool_choice", case.tool_call and model.shape != "bedrock_converse"), ("modalities", case.audio_input or case.audio_output), ("audio", case.audio_output), ("web_search_options", case.web_search is not None), From ed0c32cdb0f399028ddb6699ec1a8544a0ba9735 Mon Sep 17 00:00:00 2001 From: kerry Date: Sat, 19 Sep 2026 04:04:40 +0000 Subject: [PATCH 30/30] test(integration): drive cost tracking from literal request/response data Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/README.md | 2 +- tests/integration/_support/scripted_shapes.py | 1364 - tests/integration/_support/upstream.py | 172 +- tests/integration/contracts.json | 726 +- tests/integration/cost_calculation/cases.json | 2958 -- .../integration/cost_calculation/conftest.py | 28 +- .../cost_calculation/cost_map.json | 411 - .../cost_calculation/cost_matrix.py | 658 - .../cost_calculation/cost_tracking_case.py | 253 + .../cost_calculation/cost_tracking_cases.json | 25658 ++++++++++++++++ .../cost_calculation/test_cost_tracking.py | 101 + .../cost_calculation/test_token_pricing.py | 245 - 12 files changed, 26495 insertions(+), 6081 deletions(-) delete mode 100644 tests/integration/_support/scripted_shapes.py delete mode 100644 tests/integration/cost_calculation/cases.json delete mode 100644 tests/integration/cost_calculation/cost_map.json delete mode 100644 tests/integration/cost_calculation/cost_matrix.py create mode 100644 tests/integration/cost_calculation/cost_tracking_case.py create mode 100644 tests/integration/cost_calculation/cost_tracking_cases.json create mode 100644 tests/integration/cost_calculation/test_cost_tracking.py delete mode 100644 tests/integration/cost_calculation/test_token_pricing.py diff --git a/tests/integration/README.md b/tests/integration/README.md index dcdf0e9fa96..f21e04f1ca5 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -2,7 +2,7 @@ These tests exercise a running gateway, PostgreSQL and Redis with an owned local upstream. CircleCI owns this suite. Tests are grouped by behavior, with no automatic test retries or fallback to paid provider calls -The `cost` group runs the scripted-shape cost matrix through the shared integration upstream, which serves the test-owned cost map over loopback through `LITELLM_MODEL_COST_MAP_URL`; cost goldens are checked into the integration suite and must not be copied into the E2E coverage registry. The upstream renders a scenario in the shape LiteLLM's own provider config resolves to for the deployment, so a provider LiteLLM already parses with one of the five rendered families is a cost-map entry plus a `cases.json` `providers` row with its deployment parameters; a provider whose config class is none of those families fails at collection until `scripted_shapes.py` gains a renderer +The `cost` group is driven by `cost_tracking_cases.json`, which contains the cost map, literal requests, literal provider responses and expected accounting values. Each case has a name, contract ID, cost-map model, optional deployment overrides, request body, tagged response and exact or recount expectations. Request bodies use `$MODEL` for the registered proxy model, while responses use `$REQUEST_ID` for the per-run scenario ID. To add a case, add a cost-map entry when the model is new, add the request body and exact provider response data, add hand-computed expected values and register the node ID in `contracts.json`. The upstream serves each stored response for any path under `/`, while the test-owned cost map is served over loopback through `LITELLM_MODEL_COST_MAP_URL` Use `tests/integration/run.py management`, `accounting`, `database`, `providers`, `extensions`, `sdk` or `cost` to run a selected group. Set `INTEGRATION_PROXY_URL`, `INTEGRATION_UPSTREAM_URL`, `INTEGRATION_MASTER_KEY` and `DATABASE_URL` to an isolated test deployment. The runner selects the new domain directories explicitly; the legacy OCI and sandbox selections remain separate diff --git a/tests/integration/_support/scripted_shapes.py b/tests/integration/_support/scripted_shapes.py deleted file mode 100644 index 61bfe7c24f1..00000000000 --- a/tests/integration/_support/scripted_shapes.py +++ /dev/null @@ -1,1364 +0,0 @@ -"""Scripted response shapes for the cost-calculation integration suite. - -This module owns the Scenario schema, the five renderers, one per LiteLLM -parser family, and the dispatcher. Because the usage is scripted, expected -spend is literal arithmetic on the test cost map's rates, with no dependency -on what a real provider would report. - -The upstream exposes: - -- ``POST /__scenarios`` register a Scenario JSON, returns its id -- ``DELETE /__scenarios/`` remove it -- ``POST //`` provider response; the remainder is whatever - path the provider client appends (``chat/completions``, ``responses``, - ``v1/messages``, ``models/:generateContent`` ...). Vertex appends - ``:generateContent`` / ``:streamGenerateContent`` to the scenario segment, - and Bedrock Converse targets ``model//converse`` / - ``converse-stream`` - -A request carrying ``"stream": true`` (or the ``:streamGenerateContent`` Gemini -verb) gets an SSE answer; ``stream_usage`` on the Scenario decides whether the -final stream chunk carries usage or the provider reports none. -""" - -from __future__ import annotations - -import json -import struct -import threading -import time -import zlib -from collections.abc import Mapping -from dataclasses import dataclass -from types import MappingProxyType -from typing import Final, Literal, TypeAlias, assert_never -from urllib.parse import unquote, urlsplit - -from pydantic import BaseModel, ConfigDict, TypeAdapter, model_validator - -Shape: TypeAlias = Literal[ - "openai_chat", - "openai_responses", - "anthropic_messages", - "gemini_generate", - "bedrock_converse", -] - - -@dataclass(frozen=True, slots=True) -class ShapeSpec: - usage: frozenset[str] - terminals: frozenset[str] - - -SHAPES: Final[Mapping[Shape, ShapeSpec]] = MappingProxyType( - { - "openai_chat": ShapeSpec( - usage=frozenset( - { - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "web_search_calls", - } - ), - terminals=frozenset(), - ), - "openai_responses": ShapeSpec( - usage=frozenset( - { - "cache_read_tokens", - "reasoning_tokens", - "web_search_calls", - "file_search_calls", - } - ), - terminals=frozenset({"incomplete", "unvalidated"}), - ), - "anthropic_messages": ShapeSpec( - usage=frozenset( - { - "cache_read_tokens", - "web_search_calls", - "cache_write_5m_tokens", - "cache_write_1h_tokens", - } - ), - terminals=frozenset(), - ), - "gemini_generate": ShapeSpec( - usage=frozenset( - { - "cache_read_tokens", - "reasoning_tokens", - "audio_input_tokens", - "audio_output_tokens", - "image_input_tokens", - "video_input_tokens", - "web_search_calls", - "google_maps_calls", - } - ), - terminals=frozenset({"prompt_blocked"}), - ), - "bedrock_converse": ShapeSpec( - usage=frozenset( - { - "cache_read_tokens", - "cache_write_5m_tokens", - "cache_write_1h_tokens", - } - ), - terminals=frozenset(), - ), - } -) -StreamUsage: TypeAlias = Literal["final_chunk", "absent"] -ServiceTier: TypeAlias = Literal["flex", "priority"] -TerminalKind: TypeAlias = Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] - -_BASE_USAGE_FIELDS: Final = frozenset({"fresh_input_tokens", "output_tokens"}) - - -class ScriptedToolCall(BaseModel): - """A single function call the scripted output emits instead of text. - ``arguments`` is the shape's JSON string (~250 chars), sliced into deltas - for streams.""" - - model_config = ConfigDict(frozen=True) - - name: str - arguments: str - - -class ScriptedUsage(BaseModel): - """Physical token counts the scripted response reports. ``fresh_input_tokens`` - is the uncached, never-written, non-audio input count; ``output_tokens`` is - the non-reasoning, non-audio output count. Renderers add the cached, written, - audio, and reasoning counts into the shape's total fields the way the real - provider does (inside prompt_tokens for OpenAI/Gemini, as uncached-only - input_tokens for Anthropic).""" - - model_config = ConfigDict(frozen=True) - - fresh_input_tokens: int = 0 - output_tokens: int = 0 - cache_read_tokens: int = 0 - cache_write_5m_tokens: int = 0 - cache_write_1h_tokens: int = 0 - reasoning_tokens: int = 0 - audio_input_tokens: int = 0 - audio_output_tokens: int = 0 - image_input_tokens: int = 0 - video_input_tokens: int = 0 - web_search_calls: int = 0 - google_maps_calls: int = 0 - file_search_calls: int = 0 - - -class ScriptedOutput(BaseModel): - model_config = ConfigDict(frozen=True) - - text: str - finish_reason: str = "stop" - # When set, emitted verbatim as the response's model field, letting a test - # prove the biller prices the provider-reported model. - response_model: str | None = None - # OpenAI-compatible providers can report a provider-computed cost; emitted as - # the top-level "cost" field on the together/fireworks response. - provider_cost: float | None = None - # When set, the response is a tool call only: no text content on any response. - tool_call: ScriptedToolCall | None = None - # Terminal shape: "unvalidated" makes the Responses terminal response fail - # pydantic validation so the proxy takes its model_construct dict path; - # "prompt_blocked" is a Gemini promptFeedback-only body. - terminal: TerminalKind = "completed" - - -class Scenario(BaseModel): - model_config = ConfigDict(frozen=True) - - scenario_id: str - shape: Shape - usage: ScriptedUsage - output: ScriptedOutput - # The bare provider-facing model name the renderer echoes when the request - # carries no model of its own (Vertex and Bedrock name the model in the URL - # path, not the body). - model: str - stream_usage: StreamUsage = "final_chunk" - service_tier: ServiceTier | None = None - # Anthropic fast mode and US inference geography; emitted on the anthropic - # usage object only (litellm reads them there), so they are response-side. - speed: Literal["fast"] | None = None - inference_geo: Literal["us"] | None = None - - @model_validator(mode="after") - def _check_terminal_supported(self) -> Scenario: - spec: Final = SHAPES[self.shape] - if ( - self.output.terminal != "completed" - and self.output.terminal not in spec.terminals - ): - raise ValueError( - f"shape {self.shape} cannot emit terminal={self.output.terminal}" - ) - unsupported: Final = frozenset( - field - for field in self.usage.model_fields_set - if getattr(self.usage, field) - and field not in (spec.usage | _BASE_USAGE_FIELDS) - ) - if unsupported: - raise ValueError( - f"shape {self.shape} cannot express usage fields {sorted(unsupported)}" - ) - if (self.speed or self.inference_geo) and self.shape != "anthropic_messages": - raise ValueError( - f"shape {self.shape} cannot emit speed/inference_geo (anthropic usage fields)" - ) - return self - - -class ScenarioRegistered(BaseModel): - scenario_id: str - - -class ScenarioDeleted(BaseModel): - deleted: bool - - -class HealthStatus(BaseModel): - status: str - - -@dataclass(frozen=True, slots=True) -class RenderedResponse: - status_code: int - content_type: str - body: bytes - - -def _jobj(*pairs: tuple[str, object]) -> Mapping[str, object]: - """A JSON object payload built in one shot and frozen.""" - return MappingProxyType(dict(pairs)) - - -def _jobj_opt(*pairs: tuple[str, object] | None) -> Mapping[str, object]: - """``_jobj`` where a ``None`` pair means the field is absent.""" - return MappingProxyType(dict(pair for pair in pairs if pair is not None)) - - -def _json_bytes(payload: Mapping[str, object]) -> bytes: - return json.dumps(payload, default=dict).encode("utf-8") - - -def _sse_frame(event_name: str | None, data: Mapping[str, object] | str) -> str: - head: Final = f"event: {event_name}\n" if event_name is not None else "" - payload: Final = data if isinstance(data, str) else json.dumps(data, default=dict) - return f"{head}data: {payload}\n\n" - - -def _sse(events: tuple[tuple[str | None, Mapping[str, object] | str], ...]) -> bytes: - return "".join(_sse_frame(event_name, data) for event_name, data in events).encode("utf-8") - - - # ---------- per-shape usage shapes ---------- - - -def _openai_usage(u: ScriptedUsage) -> Mapping[str, object]: - prompt_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - completion_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - prompt_details: Final = _jobj_opt( - ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, - ("audio_tokens", u.audio_input_tokens) if u.audio_input_tokens else None, - ) - completion_details: Final = _jobj_opt( - ("reasoning_tokens", u.reasoning_tokens) if u.reasoning_tokens else None, - ("audio_tokens", u.audio_output_tokens) if u.audio_output_tokens else None, - ) - return _jobj_opt( - ("prompt_tokens", prompt_tokens), - ("completion_tokens", completion_tokens), - ("total_tokens", prompt_tokens + completion_tokens), - ("prompt_tokens_details", prompt_details) if prompt_details else None, - ("completion_tokens_details", completion_details) if completion_details else None, - ) - - -def _anthropic_usage(scenario: Scenario) -> Mapping[str, object]: - # Anthropic reports uncached-only input_tokens; cache reads and writes ride - # top-level fields, with the 5m/1h write split under cache_creation. - u: Final = scenario.usage - return _jobj_opt( - ("input_tokens", u.fresh_input_tokens), - ("output_tokens", u.output_tokens), - ("service_tier", scenario.service_tier) if scenario.service_tier else None, - ("speed", scenario.speed) if scenario.speed else None, - ("inference_geo", scenario.inference_geo) if scenario.inference_geo else None, - ("cache_read_input_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, - ( - ("cache_creation_input_tokens", u.cache_write_5m_tokens + u.cache_write_1h_tokens) - if u.cache_write_5m_tokens or u.cache_write_1h_tokens - else None - ), - ( - ( - "cache_creation", - _jobj( - ("ephemeral_5m_input_tokens", u.cache_write_5m_tokens), - ("ephemeral_1h_input_tokens", u.cache_write_1h_tokens), - ), - ) - if u.cache_write_5m_tokens or u.cache_write_1h_tokens - else None - ), - ( - ("server_tool_use", _jobj(("web_search_requests", u.web_search_calls))) - if u.web_search_calls - else None - ), - ) - - -def _gemini_usage(scenario: Scenario) -> Mapping[str, object]: - # Real generateContent accounting: promptTokenCount carries the cached count - # inside it (TEXT modality is the cached-inclusive text count so litellm's - # implicit-caching subtraction lands on the fresh figure), candidatesTokenCount - # excludes thoughts, thoughtsTokenCount reports them separately, and - # totalTokenCount sums all three. Image/video input ride promptTokensDetails. - u: Final = scenario.usage - prompt_tokens: Final = ( - u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - + u.image_input_tokens + u.video_input_tokens - ) - candidates: Final = u.output_tokens + u.audio_output_tokens - return _jobj_opt( - ("promptTokenCount", prompt_tokens), - ("candidatesTokenCount", candidates), - ("thoughtsTokenCount", u.reasoning_tokens) if u.reasoning_tokens else None, - ("totalTokenCount", prompt_tokens + candidates + u.reasoning_tokens), - ("cachedContentTokenCount", u.cache_read_tokens) if u.cache_read_tokens else None, - ( - "promptTokensDetails", - ( - _jobj(("modality", "TEXT"), ("tokenCount", u.fresh_input_tokens + u.cache_read_tokens)), - *( - (_jobj(("modality", "AUDIO"), ("tokenCount", u.audio_input_tokens)),) - if u.audio_input_tokens - else () - ), - *( - (_jobj(("modality", "IMAGE"), ("tokenCount", u.image_input_tokens)),) - if u.image_input_tokens - else () - ), - *( - (_jobj(("modality", "VIDEO"), ("tokenCount", u.video_input_tokens)),) - if u.video_input_tokens - else () - ), - ), - ), - ( - ( - "candidatesTokensDetails", - ( - _jobj(("modality", "TEXT"), ("tokenCount", u.output_tokens)), - _jobj(("modality", "AUDIO"), ("tokenCount", u.audio_output_tokens)), - ), - ) - if u.audio_output_tokens - else None - ), - ( - ( - "trafficType", - {"flex": "ON_DEMAND_FLEX", "priority": "ON_DEMAND_PRIORITY"}[ - scenario.service_tier - ], - ) - if scenario.service_tier - else None - ), - ) - - -def _gemini_grounding_metadata(scenario: Scenario) -> Mapping[str, object] | None: - """groundingMetadata for the search/Maps flags. Maps items carry maps - chunks and googleMapsWidgetContextToken so litellm bills them as Maps - queries, not web search.""" - u: Final = scenario.usage - if not u.web_search_calls and not u.google_maps_calls: - return None - if u.google_maps_calls: - return _jobj( - ( - "webSearchQueries", - tuple(f"maps query {i}" for i in range(u.google_maps_calls)), - ), - ( - "groundingChunks", - tuple( - _jobj(("maps", _jobj(("uri", f"https://maps.google.com/?cid={i}")))) - for i in range(u.google_maps_calls) - ), - ), - ("googleMapsWidgetContextToken", f"token_{scenario.scenario_id}"), - ) - return _jobj( - ("webSearchQueries", tuple(f"query {i}" for i in range(u.web_search_calls))), - ) - - -def _responses_usage(u: ScriptedUsage) -> Mapping[str, object]: - input_tokens: Final = u.fresh_input_tokens + u.cache_read_tokens + u.audio_input_tokens - output_tokens: Final = u.output_tokens + u.reasoning_tokens + u.audio_output_tokens - input_details: Final = _jobj_opt( - ("cached_tokens", u.cache_read_tokens) if u.cache_read_tokens else None, - ) - return _jobj_opt( - ("input_tokens", input_tokens), - ("output_tokens", output_tokens), - ("total_tokens", input_tokens + output_tokens), - ("input_tokens_details", input_details) if input_details else None, - ( - ("output_tokens_details", _jobj(("reasoning_tokens", u.reasoning_tokens))) - if u.reasoning_tokens - else None - ), - ) - - - # ---------- per-shape responses ---------- - - -def _split_arguments(arguments: str) -> tuple[str, ...]: - """Slice a tool-call arguments JSON string into 2-3 streamed deltas.""" - third: Final = max(1, len(arguments) // 3) - return tuple( - slice_ - for slice_ in (arguments[:third], arguments[third : 2 * third], arguments[2 * third :]) - if slice_ - ) - - -def _openai_message(scenario: Scenario) -> Mapping[str, object]: - tool_call: Final = scenario.output.tool_call - return _jobj_opt( - ("role", "assistant"), - ("content", None if tool_call is not None else scenario.output.text), - ( - ( - "tool_calls", - ( - _jobj( - ("id", f"call_{scenario.scenario_id}"), - ("type", "function"), - ( - "function", - _jobj(("name", tool_call.name), ("arguments", tool_call.arguments)), - ), - ), - ), - ) - if tool_call is not None - else None - ), - ( - ( - "annotations", - tuple( - _jobj( - ("type", "url_citation"), - ( - "url_citation", - _jobj( - ("url", "https://scripted.example/source"), - ("title", "scripted source"), - ("start_index", 0), - ("end_index", 1), - ), - ), - ) - for _ in range(scenario.usage.web_search_calls) - ), - ) - if scenario.usage.web_search_calls - else None - ), - ) - - -def _openai_chat_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - return _jobj_opt( - ("id", f"chatcmpl-{scenario.scenario_id}"), - ("object", "chat.completion"), - ("created", int(time.time())), - ("model", scenario.output.response_model or requested_model), - ( - "choices", - ( - _jobj( - ("index", 0), - ("message", _openai_message(scenario)), - ( - "finish_reason", - "tool_calls" - if scenario.output.tool_call is not None - else scenario.output.finish_reason, - ), - ), - ), - ), - ("usage", _openai_usage(scenario.usage)), - ("service_tier", scenario.service_tier) if scenario.service_tier is not None else None, - ("cost", scenario.output.provider_cost) if scenario.output.provider_cost is not None else None, - ) - - -def _openai_chunk( - scenario: Scenario, - requested_model: str, - choices: tuple[Mapping[str, object], ...] = (), - usage: Mapping[str, object] | None = None, -) -> Mapping[str, object]: - return _jobj_opt( - ("id", f"chatcmpl-{scenario.scenario_id}"), - ("object", "chat.completion.chunk"), - ("created", int(time.time())), - ("model", scenario.output.response_model or requested_model), - ("choices", choices), - ("usage", usage), - ) - - -def _openai_chat_sse(scenario: Scenario, requested_model: str) -> bytes: - tool_call: Final = scenario.output.tool_call - delta: Final = _jobj_opt( - ("role", "assistant"), - ("content", scenario.output.text), - ( - ("annotations", _openai_message(scenario)["annotations"]) - if scenario.usage.web_search_calls - else None - ), - ) - body_deltas: Final[tuple[Mapping[str, object], ...]] = ( - ( - _jobj( - ("role", "assistant"), - ( - "tool_calls", - ( - _jobj( - ("index", 0), - ("id", f"call_{scenario.scenario_id}"), - ("type", "function"), - ( - "function", - _jobj(("name", tool_call.name), ("arguments", "")), - ), - ), - ), - ), - ), - *( - _jobj( - ( - "tool_calls", - ( - _jobj( - ("index", 0), - ("function", _jobj(("arguments", arguments_slice))), - ), - ), - ) - ) - for arguments_slice in _split_arguments(tool_call.arguments) - ), - ) - if tool_call is not None - else (delta,) - ) - return _sse( - ( - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=(_jobj(("index", 0), ("delta", _jobj(("role", "assistant"))), ("finish_reason", None)),), - ), - ), - *( - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=(_jobj(("index", 0), ("delta", body_delta), ("finish_reason", None)),), - ), - ) - for body_delta in body_deltas - ), - ( - None, - _openai_chunk( - scenario, - requested_model, - choices=( - _jobj( - ("index", 0), - ("delta", _jobj()), - ( - "finish_reason", - "tool_calls" - if tool_call is not None - else scenario.output.finish_reason, - ), - ), - ), - ), - ), - *( - ((None, _openai_chunk(scenario, requested_model, usage=_openai_usage(scenario.usage))),) - if scenario.stream_usage == "final_chunk" - else () - ), - (None, "[DONE]"), - ) - ) - - -def _anthropic_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: - tool_call: Final = scenario.output.tool_call - if tool_call is not None: - return ( - _jobj( - ("type", "tool_use"), - ("id", f"toolu_{scenario.scenario_id}"), - ("name", tool_call.name), - ("input", json.loads(tool_call.arguments)), - ), - ) - return (_jobj(("type", "text"), ("text", scenario.output.text)),) - - -def _anthropic_stop_reason(scenario: Scenario) -> str: - if scenario.output.tool_call is not None: - return "tool_use" - return "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason - - -def _anthropic_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - return _jobj( - ("id", f"msg_{scenario.scenario_id}"), - ("type", "message"), - ("role", "assistant"), - ("model", scenario.output.response_model or requested_model), - ("content", _anthropic_content(scenario)), - ("stop_reason", _anthropic_stop_reason(scenario)), - ("usage", _anthropic_usage(scenario)), - ) - - -def _anthropic_sse(scenario: Scenario, requested_model: str) -> bytes: - emit_usage: Final = scenario.stream_usage == "final_chunk" - input_usage: Final = _jobj( - *( - (key, value) - for key, value in _anthropic_usage(scenario).items() - if key != "output_tokens" - ) - ) - message_start: Final = _jobj( - ("type", "message_start"), - ( - "message", - _jobj_opt( - ("id", f"msg_{scenario.scenario_id}"), - ("type", "message"), - ("role", "assistant"), - ("model", scenario.output.response_model or requested_model), - ("content", ()), - ("stop_reason", None), - ("usage", input_usage) if emit_usage else None, - ), - ), - ) - message_delta: Final = _jobj_opt( - ("type", "message_delta"), - ( - "delta", - _jobj(("stop_reason", _anthropic_stop_reason(scenario))), - ), - ( - ("usage", _jobj(("output_tokens", scenario.usage.output_tokens))) - if emit_usage - else None - ), - ) - return _sse( - ( - ("message_start", message_start), - ( - "content_block_start", - _jobj( - ("type", "content_block_start"), - ("index", 0), - ( - "content_block", - _jobj( - ("type", "tool_use"), - ("id", f"toolu_{scenario.scenario_id}"), - ("name", scenario.output.tool_call.name), - ("input", _jobj()), - ) - if scenario.output.tool_call is not None - else _jobj(("type", "text"), ("text", "")), - ), - ), - ), - *( - tuple( - ( - "content_block_delta", - _jobj( - ("type", "content_block_delta"), - ("index", 0), - ( - "delta", - _jobj(("type", "input_json_delta"), ("partial_json", arguments_slice)), - ), - ), - ) - for arguments_slice in _split_arguments(scenario.output.tool_call.arguments) - ) - if scenario.output.tool_call is not None - else ( - ( - "content_block_delta", - _jobj( - ("type", "content_block_delta"), - ("index", 0), - ("delta", _jobj(("type", "text_delta"), ("text", scenario.output.text))), - ), - ), - ) - ), - ("content_block_stop", _jobj(("type", "content_block_stop"), ("index", 0))), - ("message_delta", message_delta), - ("message_stop", _jobj(("type", "message_stop"))), - ) - ) - - -def _gemini_prompt_blocked_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - return _jobj( - ( - "promptFeedback", - _jobj( - ("blockReason", "SAFETY"), - ( - "safetyRatings", - ( - _jobj( - ("category", "HARM_CATEGORY_HARASSMENT"), - ("probability", "HIGH"), - ("blocked", True), - ), - ), - ), - ), - ), - ("usageMetadata", _gemini_usage(scenario)), - ("modelVersion", scenario.output.response_model or requested_model), - ) - - -def _gemini_parts(scenario: Scenario) -> tuple[Mapping[str, object], ...]: - tool_call: Final = scenario.output.tool_call - if tool_call is not None: - return ( - _jobj( - ( - "functionCall", - _jobj( - ("name", tool_call.name), - ("args", json.loads(tool_call.arguments)), - ), - ) - ), - ) - return (_jobj(("text", scenario.output.text)),) - - -def _gemini_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - if scenario.output.terminal == "prompt_blocked": - return _gemini_prompt_blocked_body(scenario, requested_model) - return _jobj( - ( - "candidates", - ( - _jobj_opt( - ( - "content", - _jobj( - ("parts", _gemini_parts(scenario)), - ("role", "model"), - ), - ), - ( - "finishReason", - "STOP" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason.upper(), - ), - ("index", 0), - ( - ("groundingMetadata", _gemini_grounding_metadata(scenario)) - if _gemini_grounding_metadata(scenario) is not None - else None - ), - ), - ), - ), - ("usageMetadata", _gemini_usage(scenario)), - ("modelVersion", scenario.output.response_model or requested_model), - ) - - -def _gemini_sse(scenario: Scenario, requested_model: str) -> bytes: - emit_usage: Final = scenario.stream_usage == "final_chunk" - first: Final = _jobj( - *((key, value) for key, value in _gemini_body(scenario, requested_model).items() if key != "usageMetadata") - ) - return _sse( - ( - (None, first), - *( - ( - ( - None, - _jobj( - ("candidates", ()), - ("usageMetadata", _gemini_usage(scenario)), - ("modelVersion", scenario.output.response_model or requested_model), - ), - ), - ) - if emit_usage - else () - ), - ) - ) - - -def _responses_output(scenario: Scenario) -> tuple[Mapping[str, object], ...]: - tool_call: Final = scenario.output.tool_call - return ( - *( - ( - _jobj(("type", "scripted_future_item"), ("id", f"fut_{scenario.scenario_id}"), ("status", "completed")), - ) - if scenario.output.terminal == "unvalidated" - else () - ), - *( - _jobj(("type", "web_search_call"), ("id", f"ws_{i}"), ("status", "completed")) - for i in range(scenario.usage.web_search_calls) - ), - *( - _jobj( - ("type", "file_search_call"), - ("id", f"fs_{i}"), - ("status", "completed"), - ("queries", (f"query {i}",)), - ("results", ()), - ) - for i in range(scenario.usage.file_search_calls) - ), - _jobj( - ("type", "function_call"), - ("id", f"fc_{scenario.scenario_id}"), - ("call_id", f"call_{scenario.scenario_id}"), - ("name", tool_call.name), - ("arguments", tool_call.arguments), - ("status", "completed"), - ) - if tool_call is not None - else _jobj( - ("type", "message"), - ("id", f"msg_{scenario.scenario_id}"), - ("status", "completed"), - ("role", "assistant"), - ( - "content", - ( - _jobj( - ("type", "output_text"), - ("text", scenario.output.text), - ("annotations", ()), - ), - ), - ), - ), - ) - - -def _responses_body(scenario: Scenario, requested_model: str) -> Mapping[str, object]: - incomplete: Final = scenario.output.terminal == "incomplete" - return _jobj_opt( - ("id", f"resp_{scenario.scenario_id}"), - ("object", "response"), - ( - "created_at", - "not-a-number" if scenario.output.terminal == "unvalidated" else int(time.time()), - ), - ("status", "incomplete" if incomplete else "completed"), - ( - ("incomplete_details", _jobj(("reason", "max_output_tokens"))) - if incomplete - else None - ), - ("model", scenario.output.response_model or requested_model), - ("output", _responses_output(scenario)), - ("usage", _responses_usage(scenario.usage)), - ) - - -def _responses_sse(scenario: Scenario, requested_model: str) -> bytes: - tool_call: Final = scenario.output.tool_call - terminal: Final = ( - _jobj(*((key, value) for key, value in _responses_body(scenario, requested_model).items() if key != "usage")) - if scenario.stream_usage == "absent" - else _responses_body(scenario, requested_model) - ) - created: Final = _jobj( - *((key, value) for key, value in terminal.items() if key not in ("status", "usage")), - ("status", "in_progress"), - ("usage", None), - ) - terminal_event: Final = ( - "response.incomplete" if scenario.output.terminal == "incomplete" else "response.completed" - ) - output_index: Final = ( - scenario.usage.web_search_calls - + scenario.usage.file_search_calls - + (1 if scenario.output.terminal == "unvalidated" else 0) - ) - file_search_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = tuple( - event - for i in range(scenario.usage.file_search_calls) - for event in ( - ( - "response.output_item.added", - _jobj( - ("type", "response.output_item.added"), - ("output_index", i), - ( - "item", - _jobj( - ("type", "file_search_call"), - ("id", f"fs_{i}"), - ("status", "in_progress"), - ("queries", ()), - ), - ), - ), - ), - ( - "response.output_item.done", - _jobj( - ("type", "response.output_item.done"), - ("output_index", i), - ( - "item", - _jobj( - ("type", "file_search_call"), - ("id", f"fs_{i}"), - ("status", "completed"), - ("queries", (f"query {i}",)), - ("results", ()), - ), - ), - ), - ), - ) - ) - call_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( - ( - ( - "response.output_item.added", - _jobj( - ("type", "response.output_item.added"), - ("output_index", output_index), - ( - "item", - _jobj( - ("type", "function_call"), - ("id", f"fc_{scenario.scenario_id}"), - ("call_id", f"call_{scenario.scenario_id}"), - ("name", tool_call.name), - ("arguments", ""), - ("status", "in_progress"), - ), - ), - ), - ), - *( - ( - "response.function_call_arguments.delta", - _jobj( - ("type", "response.function_call_arguments.delta"), - ("item_id", f"fc_{scenario.scenario_id}"), - ("output_index", output_index), - ("delta", arguments_slice), - ), - ) - for arguments_slice in _split_arguments(tool_call.arguments) - ), - ( - "response.function_call_arguments.done", - _jobj( - ("type", "response.function_call_arguments.done"), - ("item_id", f"fc_{scenario.scenario_id}"), - ("output_index", output_index), - ("arguments", tool_call.arguments), - ), - ), - ) - if tool_call is not None - else ( - ( - "response.output_text.delta", - _jobj( - ("type", "response.output_text.delta"), - ("item_id", f"msg_{scenario.scenario_id}"), - ("output_index", output_index), - ("content_index", 0), - ("delta", scenario.output.text), - ), - ), - ) - ) - middle_events: Final[tuple[tuple[str, Mapping[str, object]], ...]] = ( - *file_search_events, - *call_events, - ) - return _sse( - ( - ("response.created", _jobj(("type", "response.created"), ("response", created))), - *middle_events, - (terminal_event, _jobj(("type", terminal_event), ("response", terminal))), - ) - ) - - -def _bedrock_usage(u: ScriptedUsage) -> Mapping[str, object]: - # Converse reports uncached input in inputTokens and rides cache reads and - # writes on top-level fields; totalTokens covers every input kind + output. - cache_writes: Final = u.cache_write_5m_tokens + u.cache_write_1h_tokens - return _jobj_opt( - ("inputTokens", u.fresh_input_tokens), - ("outputTokens", u.output_tokens), - ( - "totalTokens", - u.fresh_input_tokens + u.cache_read_tokens + cache_writes + u.output_tokens, - ), - ("cacheReadInputTokens", u.cache_read_tokens) if u.cache_read_tokens else None, - ("cacheWriteInputTokens", cache_writes) if cache_writes else None, - ( - ( - "cacheDetails", - tuple( - _jobj(("inputTokens", count), ("ttl", ttl)) - for count, ttl in ( - (u.cache_write_5m_tokens, "5m"), - (u.cache_write_1h_tokens, "1h"), - ) - if count - ), - ) - if cache_writes - else None - ), - ) - - -def _bedrock_stop_reason(scenario: Scenario) -> str: - if scenario.output.tool_call is not None: - return "tool_use" - return "end_turn" if scenario.output.finish_reason == "stop" else scenario.output.finish_reason - - -def _bedrock_content(scenario: Scenario) -> tuple[Mapping[str, object], ...]: - tool_call: Final = scenario.output.tool_call - if tool_call is not None: - return ( - _jobj( - ( - "toolUse", - _jobj( - ("toolUseId", f"tooluse_{scenario.scenario_id}"), - ("name", tool_call.name), - ("input", json.loads(tool_call.arguments)), - ), - ), - ), - ) - return (_jobj(("text", scenario.output.text)),) - - -def _bedrock_body(scenario: Scenario) -> Mapping[str, object]: - return _jobj_opt( - ( - "output", - _jobj( - ( - "message", - _jobj( - ("role", "assistant"), - ("content", _bedrock_content(scenario)), - ), - ), - ), - ), - ("stopReason", _bedrock_stop_reason(scenario)), - ("usage", _bedrock_usage(scenario.usage)), - ("metrics", _jobj(("latencyMs", 42))), - ( - ("serviceTier", _jobj(("type", scenario.service_tier))) - if scenario.service_tier - else None - ), - ) - - -def _aws_str_header(name: str, value: str) -> bytes: - """One eventstream header: 1-byte name len + name + type-7 marker + value.""" - name_b: Final = name.encode() - value_b: Final = value.encode() - return ( - struct.pack("!B", len(name_b)) - + name_b - + struct.pack("!B", 7) - + struct.pack("!H", len(value_b)) - + value_b - ) - - -def _aws_event_frame(event_type: str, payload: Mapping[str, object]) -> bytes: - """One application/vnd.amazon.eventstream frame: prelude + prelude CRC32 + - headers + JSON payload + message CRC32, matching botocore EventStreamBuffer.""" - payload_bytes: Final = json.dumps(payload, default=dict, separators=(",", ":")).encode() - headers_bytes: Final = ( - _aws_str_header(":event-type", event_type) - + _aws_str_header(":content-type", "application/json") - + _aws_str_header(":message-type", "event") - ) - total_length: Final = 12 + len(headers_bytes) + len(payload_bytes) + 4 - prelude: Final = struct.pack("!II", total_length, len(headers_bytes)) - prelude_crc: Final = struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) - message: Final = prelude + prelude_crc + headers_bytes + payload_bytes - return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF) - - -def _bedrock_eventstream(scenario: Scenario) -> bytes: - tool_call: Final = scenario.output.tool_call - block_start: Final[tuple[bytes, ...]] = ( - ( - _aws_event_frame( - "contentBlockStart", - _jobj( - ( - "start", - _jobj( - ( - "toolUse", - _jobj( - ("toolUseId", f"tooluse_{scenario.scenario_id}"), - ("name", tool_call.name), - ), - ), - ), - ), - ("contentBlockIndex", 0), - ), - ), - ) - if tool_call is not None - else () - ) - deltas: Final[tuple[bytes, ...]] = ( - tuple( - _aws_event_frame( - "contentBlockDelta", - _jobj( - ("delta", _jobj(("toolUse", _jobj(("input", arguments_slice))))), - ("contentBlockIndex", 0), - ), - ) - for arguments_slice in _split_arguments(tool_call.arguments) - ) - if tool_call is not None - else ( - _aws_event_frame( - "contentBlockDelta", - _jobj( - ("delta", _jobj(("text", scenario.output.text))), - ("contentBlockIndex", 0), - ), - ), - ) - ) - return b"".join( - ( - _aws_event_frame("messageStart", _jobj(("role", "assistant"))), - *block_start, - *deltas, - _aws_event_frame("contentBlockStop", _jobj(("contentBlockIndex", 0))), - _aws_event_frame("messageStop", _jobj(("stopReason", _bedrock_stop_reason(scenario)))), - *( - ( - _aws_event_frame( - "metadata", - _jobj_opt( - ("usage", _bedrock_usage(scenario.usage)), - ("metrics", _jobj(("latencyMs", 42))), - ( - ("serviceTier", _jobj(("type", scenario.service_tier))) - if scenario.service_tier - else None - ), - ), - ), - ) - if scenario.stream_usage == "final_chunk" - else () - ), - ) - ) - - -def _render( - scenario: Scenario, *, stream: bool, requested_model: str, path_tail: str -) -> RenderedResponse: - # Azure bridges gpt-5.4+ chat requests carrying function tools onto the - # Responses API, which lands on the same shape at openai/responses. - if scenario.shape == "openai_chat" and path_tail.endswith("openai/responses"): - if stream: - return RenderedResponse( - 200, "text/event-stream", _responses_sse(scenario, requested_model) - ) - return RenderedResponse( - 200, "application/json", _json_bytes(_responses_body(scenario, requested_model)) - ) - shape: Final = scenario.shape - match shape: - case "bedrock_converse": - if stream: - return RenderedResponse( - 200, "application/vnd.amazon.eventstream", _bedrock_eventstream(scenario) - ) - return RenderedResponse(200, "application/json", _json_bytes(_bedrock_body(scenario))) - case "gemini_generate": - if stream: - return RenderedResponse(200, "text/event-stream", _gemini_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_gemini_body(scenario, requested_model))) - case "anthropic_messages": - if stream: - return RenderedResponse(200, "text/event-stream", _anthropic_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_anthropic_body(scenario, requested_model))) - case "openai_responses": - if stream: - return RenderedResponse(200, "text/event-stream", _responses_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_responses_body(scenario, requested_model))) - case "openai_chat": - if stream: - return RenderedResponse(200, "text/event-stream", _openai_chat_sse(scenario, requested_model)) - return RenderedResponse(200, "application/json", _json_bytes(_openai_chat_body(scenario, requested_model))) - case _: - assert_never(shape) - - -# ---------- registry + request routing ---------- - - -class ScenarioStore: - def __init__(self) -> None: - self._lock: Final = threading.Lock() - self._scenarios: dict[str, Scenario] = {} # mutable-ok: server state, guarded by _lock - - def put(self, scenario: Scenario) -> None: - with self._lock: - self._scenarios[scenario.scenario_id] = scenario - - def drop(self, scenario_id: str) -> bool: - with self._lock: - return self._scenarios.pop(scenario_id, None) is not None - - def get(self, scenario_id: str) -> Scenario | None: - with self._lock: - return self._scenarios.get(scenario_id) - - -_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) - - -def _request_body(body: bytes) -> Mapping[str, object]: - try: - return _REQUEST_BODY.validate_json(body) - except ValueError: - return MappingProxyType({}) - - -def _request_wants_stream(endpoint: str | None, path_tail: str, body: bytes) -> bool: - if endpoint == "streamGenerateContent" or ":streamGenerateContent" in path_tail: - return True - if path_tail.endswith("converse-stream"): - return True - if not body: - return False - return _request_body(body).get("stream") is True - - -def _request_model(body: bytes, path_tail: str, scenario: Scenario) -> str: - model: Final = _request_body(body).get("model") - if isinstance(model, str): - return model - # Bedrock Converse names the model in the path: model//converse[-stream]. - if path_tail.startswith("model/"): - path_model: Final = path_tail.split("/", 2)[1] if path_tail.count("/") >= 2 else "" - if path_model: - return unquote(path_model) - # Vertex names it in the URL too, but the path may carry only the endpoint; - # fall back to the scenario's declared model. - return scenario.model - - -def render(store: ScenarioStore, method: str, raw_path: str, body: bytes) -> RenderedResponse: - path: Final = urlsplit(raw_path).path - segments: Final = tuple(segment for segment in path.split("/") if segment) - if len(segments) < 1 or method != "POST": - return RenderedResponse( - 404, "application/json", _json_bytes(_jobj(("error", f"no route for {method} {path}"))) - ) - scenario_segment: Final = segments[0] - scenario_id, endpoint = ( - scenario_segment.split(":", 1) - if ":" in scenario_segment - else (scenario_segment, None) - ) - found: Final = store.get(scenario_id) - if found is None: - return RenderedResponse( - 404, "application/json", _json_bytes(_jobj(("error", f"unknown scenario {scenario_id}"))) - ) - tail: Final = "/".join(segments[1:]) - return _render( - found, - stream=_request_wants_stream(endpoint, tail, body), - requested_model=_request_model(body, tail, found), - path_tail=tail, - ) diff --git a/tests/integration/_support/upstream.py b/tests/integration/_support/upstream.py index 5374d420b6a..1ad02b6a3f2 100644 --- a/tests/integration/_support/upstream.py +++ b/tests/integration/_support/upstream.py @@ -2,32 +2,34 @@ from __future__ import annotations import argparse from collections import deque +from collections.abc import Mapping import json from dataclasses import dataclass, field import os from pathlib import Path from queue import SimpleQueue +import struct from typing import Final, cast +import zlib import httpx import uvicorn -from pydantic import JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError from starlette.applications import Starlette from starlette.requests import Request from starlette.responses import JSONResponse, Response from starlette.routing import Route from _fake_openai_endpoint_server import chat_completions, completions, embeddings, health, moderations -from integration._support.scripted_shapes import ( - RenderedResponse, - Scenario, - ScenarioDeleted, - ScenarioRegistered, - ScenarioStore, - render, +from integration.cost_calculation.cost_tracking_case import ( + EventStreamResponse, + JsonResponse, + SseResponse, + StoredResponse, ) JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +CASES_FILE: Final = Path(__file__).resolve().parents[1] / "cost_calculation" / "cost_tracking_cases.json" INTERNAL_FIELDS: Final = frozenset( { "litellm_params", @@ -56,6 +58,53 @@ class Observation: body: dict[str, JsonValue] +class _ScenarioRegistration(BaseModel): + scenario_id: str + response: StoredResponse + + +def _aws_str_header(name: str, value: str) -> bytes: + name_bytes: Final = name.encode() + value_bytes: Final = value.encode() + return ( + struct.pack("!B", len(name_bytes)) + + name_bytes + + struct.pack("!B", 7) + + struct.pack("!H", len(value_bytes)) + + value_bytes + ) + + +def _aws_event_frame(event_type: str, payload: Mapping[str, JsonValue], scenario_id: str) -> bytes: + payload_bytes: Final = json.dumps(payload, separators=(",", ":")).replace( + "$REQUEST_ID", scenario_id + ).encode() + headers_bytes: Final = ( + _aws_str_header(":event-type", event_type) + + _aws_str_header(":content-type", "application/json") + + _aws_str_header(":message-type", "event") + ) + total_length: Final = 12 + len(headers_bytes) + len(payload_bytes) + 4 + prelude: Final = struct.pack("!II", total_length, len(headers_bytes)) + prelude_crc: Final = struct.pack("!I", zlib.crc32(prelude) & 0xFFFFFFFF) + message: Final = prelude + prelude_crc + headers_bytes + payload_bytes + return message + struct.pack("!I", zlib.crc32(message) & 0xFFFFFFFF) + + +class ScenarioStore: + def __init__(self) -> None: + self._scenarios: dict[str, StoredResponse] = {} + + def put(self, scenario_id: str, response: StoredResponse) -> None: + self._scenarios[scenario_id] = response + + def drop(self, scenario_id: str) -> bool: + return self._scenarios.pop(scenario_id, None) is not None + + def get(self, scenario_id: str) -> StoredResponse | None: + return self._scenarios.get(scenario_id) + + @dataclass(frozen=True, slots=True) class Provider: observations: SimpleQueue[Observation] = field(default_factory=SimpleQueue) @@ -91,7 +140,7 @@ class Provider: return await chat_completions(request) async def script(self, request: Request) -> Response: - name: Final = request.path_params["model"] + name: Final = cast(str, request.path_params["model"]) if request.method in {"DELETE", "GET"} and name not in self.scripts: return JSONResponse({"error": "Script not found"}, status_code=404) if request.method == "GET": @@ -118,71 +167,60 @@ class Provider: async def register_scenario(self, request: Request) -> Response: try: - scenario: Final = Scenario.model_validate_json(await request.body()) + registration: Final = _ScenarioRegistration.model_validate_json(await request.body()) except ValidationError as exc: - return self._render( - RenderedResponse(400, "application/json", json.dumps({"error": str(exc)}).encode("utf-8")) - ) - self.scenario_store.put(scenario) - return self._render( - RenderedResponse( - 200, - "application/json", - json.dumps({"scenario_id": scenario.scenario_id}).encode("utf-8"), - ) - ) + return JSONResponse({"error": str(exc)}, status_code=400) + self.scenario_store.put(registration.scenario_id, registration.response) + return JSONResponse({"scenario_id": registration.scenario_id}) async def delete_scenario(self, request: Request) -> Response: scenario_id: Final = cast(str, request.path_params["scenario_id"]) deleted: Final = self.scenario_store.drop(scenario_id) - return self._render( - RenderedResponse( - 200 if deleted else 404, - "application/json", - json.dumps({"deleted": deleted}).encode("utf-8"), - ) - ) + return JSONResponse({"deleted": deleted}, status_code=200 if deleted else 404) async def cost_map(self, _request: Request) -> Response: - return self._render( - RenderedResponse( - 200, - "application/json", - (Path(__file__).resolve().parents[1] / "cost_calculation" / "cost_map.json").read_bytes(), - ) - ) + cases_file: Final = JSON_OBJECT.validate_json(CASES_FILE.read_bytes()) + return JSONResponse(cases_file["cost_map"]) async def oauth_token(self, _request: Request) -> Response: - return self._render( - RenderedResponse( - 200, - "application/json", - json.dumps( - { - "access_token": "scripted-token", - "token_type": "Bearer", - "expires_in": 3600, - } - ).encode("utf-8"), - ) + return JSONResponse( + { + "access_token": "scripted-token", + "token_type": "Bearer", + "expires_in": 3600, + } ) async def scripted(self, request: Request) -> Response: - rendered: Final = render( - self.scenario_store, - request.method, - request.url.path, - await request.body(), - ) - return self._render(rendered) + segments: Final = tuple(segment for segment in cast(str, request.path_params["path"]).split("/") if segment) + if not segments: + return JSONResponse({"error": "Unknown scenario"}, status_code=404) + scenario_id: Final = segments[0].split(":", 1)[0] + response: Final = self.scenario_store.get(scenario_id) + if response is None: + return JSONResponse({"error": "Unknown scenario"}, status_code=404) + return self._response(response, scenario_id) @staticmethod - def _render(rendered: RenderedResponse) -> Response: - return Response( - content=rendered.body, - status_code=rendered.status_code, - media_type=rendered.content_type, - ) + def _response(response: StoredResponse, scenario_id: str) -> Response: + match response: + case JsonResponse(): + return Response( + content=json.dumps(response.body, separators=(",", ":")).replace( + "$REQUEST_ID", scenario_id + ).encode(), + media_type=response.content_type, + ) + case SseResponse(): + stream_body: Final = ("\n\n".join(response.frames) + "\n\n").replace( + "$REQUEST_ID", scenario_id + ) + return Response(content=stream_body.encode(), media_type=response.content_type) + case EventStreamResponse(): + event_body: Final = b"".join( + _aws_event_frame(event.event_type, event.payload, scenario_id) for event in response.events + ) + return Response(content=event_body, media_type=response.content_type) def app(self) -> Starlette: return Starlette( @@ -198,7 +236,7 @@ class Provider: Route("/v1/completions", completions, methods=["POST"]), Route("/v1/embeddings", embeddings, methods=["POST"]), Route("/v1/moderations", moderations, methods=["POST"]), - Route("/{scenario_id}/{tail:path}", self.scripted, methods=["POST"]), + Route("/{path:path}", self.scripted, methods=["POST"]), ] ) @@ -215,17 +253,16 @@ class ScenarioHandle: return f"{self.control_url}/{self.scenario_id}" -def register_scenario(scenario: Scenario) -> ScenarioHandle: - response: Final = httpx.post( +def register_scenario(scenario_id: str, response: StoredResponse) -> ScenarioHandle: + http_response: Final = httpx.post( f"{CONTROL_URL}/__scenarios", - json=scenario.model_dump(mode="json"), + json={"scenario_id": scenario_id, "response": response.model_dump(mode="json")}, trust_env=False, timeout=15, ) - response.raise_for_status() - result: Final = ScenarioRegistered.model_validate_json(response.content) + http_response.raise_for_status() return ScenarioHandle( - scenario_id=result.scenario_id, + scenario_id=scenario_id, control_url=CONTROL_URL, ) @@ -237,7 +274,6 @@ def delete_scenario(handle: ScenarioHandle) -> None: timeout=15, ) response.raise_for_status() - ScenarioDeleted.model_validate_json(response.content) def main() -> None: diff --git a/tests/integration/contracts.json b/tests/integration/contracts.json index 932ebad9fe1..8ac59516747 100644 --- a/tests/integration/contracts.json +++ b/tests/integration/contracts.json @@ -217,1093 +217,1093 @@ "tests/integration/sdk/test_http2_wire.py::test_sync_handler_negotiates_http2_only_when_enabled": [ "other.sdk_wire.http2.sync_handler_negotiates_h2_only_when_enabled" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_5m]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-cache_write_1h]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[anthropic.claude-sonnet-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-audio_output]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-audio_output]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-web_search_low]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_low]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-web_search_high]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-web_search_high]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.4-mini-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.4-mini-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-audio_output]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-audio_output]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-web_search_low]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_low]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-web_search_high]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-web_search_high]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[azure-gpt-5.6-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[azure-gpt-5.6-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-cache_write_5m]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_5m]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-cache_write_1h]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-cache_write_1h]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-anthropic_us_inference]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-anthropic_us_inference]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-haiku-4-5-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-haiku-4-5-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-cache_write_5m]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_5m]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-cache_write_1h]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-cache_write_1h]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tiered_input_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_input_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tiered_cache_read_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_read_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tiered_cache_write_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tiered_cache_write_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-anthropic_fast_mode]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_fast_mode]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-anthropic_us_inference]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-anthropic_us_inference]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-opus-5-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-opus-5-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-cache_write_5m]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_5m]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-cache_write_1h]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-cache_write_1h]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tiered_input_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_input_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tiered_cache_read_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_read_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tiered_cache_write_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tiered_cache_write_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-anthropic_us_inference]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-anthropic_us_inference]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[claude-sonnet-5-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[claude-sonnet-5-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_input_rate]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_input_rate]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-image_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-tiered_input_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_input_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-tiered_cache_read_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tiered_cache_read_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-google_maps_grounding]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-google_maps_grounding]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-fallback_video_tokens_at_input_rate]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.1-pro-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.1-pro-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-audio_output]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-audio_output]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-video_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-video_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-web_search_per_prompt]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-web_search_per_prompt]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-google_maps_grounding]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-google_maps_grounding]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-fallback_reasoning_at_output_rate]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_reasoning_at_output_rate]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-fallback_image_tokens_at_input_rate]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-3.8-flash-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-3.8-flash-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-image_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-video_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-video_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-tiered_input_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_input_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tiered_cache_read_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-google_maps_grounding]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-google_maps_grounding]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.1-pro-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.1-pro-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-audio_output]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-audio_output]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-image_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-video_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-video_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-web_search_per_prompt]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-web_search_per_prompt]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-google_maps_grounding]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-google_maps_grounding]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_prompt_blocked]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_prompt_blocked]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gemini-gemini-3.8-flash-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gemini-gemini-3.8-flash-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-web_search_low]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_low]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-web_search_high]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-web_search_high]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-file_search]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-file_search]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_incomplete]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_incomplete]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_incomplete]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_incomplete]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_unvalidated]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_unvalidated]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_no_usage_unvalidated]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_no_usage_unvalidated]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.3-codex-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.3-codex-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-audio_output]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-audio_output]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-web_search_low]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_low]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-web_search_high]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-web_search_high]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.4-mini-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.4-mini-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-web_search_low]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_low]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-web_search_high]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-web_search_high]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-file_search]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-file_search]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_incomplete]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_incomplete]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_incomplete]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_incomplete]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_unvalidated]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_unvalidated]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_no_usage_unvalidated]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_no_usage_unvalidated]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.5-pro-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.5-pro-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-audio_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_input]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-audio_output]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-audio_output]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-reasoning]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-reasoning]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-web_search_medium]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_medium]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-web_search_low]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_low]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-web_search_high]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-web_search_high]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[gpt-5.6-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[gpt-5.6-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-moonshotai-Kimi-K3-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-moonshotai-Kimi-K3-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[together_ai-zai-org-GLM-5.3-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[together_ai-zai-org-GLM-5.3-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-input_text]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-input_text]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-cache_read]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_read]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_5m]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-cache_write_1h]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_flex]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-service_tier_priority]": [ "quota_management.spend_tracking.cost_matrix.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_response_model_override]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_tool_call]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ], - "tests/integration/cost_calculation/test_token_pricing.py::test_scripted_usage_bills_at_map_rates[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ + "tests/integration/cost_calculation/test_cost_tracking.py::test_case_bills_expected_cost[us.anthropic.claude-opus-5-v1:0-stream_full_usage]": [ "quota_management.spend_tracking.scripted_wire.logs_cost" ] }, diff --git a/tests/integration/cost_calculation/cases.json b/tests/integration/cost_calculation/cases.json deleted file mode 100644 index 478aa069f1e..00000000000 --- a/tests/integration/cost_calculation/cases.json +++ /dev/null @@ -1,2958 +0,0 @@ -{ - "providers": [ - { - "litellm_provider": "openai", - "mode": "chat", - "model_prefix": "openai", - "litellm_params": {} - }, - { - "litellm_provider": "openai", - "mode": "responses", - "model_prefix": "openai/responses", - "litellm_params": {} - }, - { - "litellm_provider": "anthropic", - "mode": "chat", - "model_prefix": "anthropic", - "litellm_params": {} - }, - { - "litellm_provider": "gemini", - "mode": "chat", - "model_prefix": null, - "litellm_params": {} - }, - { - "litellm_provider": "together_ai", - "mode": "chat", - "model_prefix": null, - "litellm_params": {} - }, - { - "litellm_provider": "fireworks_ai", - "mode": "chat", - "model_prefix": null, - "litellm_params": {} - }, - { - "litellm_provider": "azure", - "mode": "chat", - "model_prefix": null, - "litellm_params": { - "api_version": "2025-04-01-preview" - } - }, - { - "litellm_provider": "bedrock_converse", - "mode": "chat", - "model_prefix": "bedrock/converse", - "litellm_params": { - "aws_access_key_id": "AKIASCRIPTEDPROVIDER", - "aws_secret_access_key": "scripted-secret", - "aws_region_name": "us-east-1" - } - }, - { - "litellm_provider": "vertex_ai-language-models", - "mode": "chat", - "model_prefix": "vertex_ai", - "litellm_params": { - "vertex_project": "cc-scripted-project", - "vertex_location": "us-central1" - } - } - ], - "deployments": [ - { - "map_key": "azure/gpt-5.4-mini", - "litellm_model": "azure/cc-pinned-deployment", - "base_model": "azure/gpt-5.4-mini" - } - ], - "cases": [ - { - "name": "input_text", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "owns": [ - "input_cost_per_token", - "output_cost_per_token" - ], - "fallback_for": [], - "expected": { - "gpt-5.6": { - "spend": 0.008988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0017976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0092448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0195, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0117, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0039, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00084124, - "input_cost": 0.0004416, - "output_cost": 0.00039964, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.008624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.002156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.0090552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.00224224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.002134, - "input_cost": 0.001104, - "output_cost": 0.00103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.0031392, - "input_cost": 0.001656, - "output_cost": 0.0014832, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "cache_read", - "family": "pricing", - "usage": { - "fresh_input_tokens": 640, - "cache_read_tokens": 12288, - "output_tokens": 380 - }, - "owns": [ - "cache_read_input_token_cost" - ], - "fallback_for": [], - "expected": { - "gpt-5.6": { - "spend": 0.0085904, - "input_cost": 0.0032704, - "output_cost": 0.00532, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gpt-5.4-mini": { - "spend": 0.00171808, - "input_cost": 0.00065408, - "output_cost": 0.001064, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "azure/gpt-5.6": { - "spend": 0.00883584, - "input_cost": 0.00336384, - "output_cost": 0.005472, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "azure/gpt-5.4-mini": { - "spend": 0.001767168, - "input_cost": 0.000672768, - "output_cost": 0.0010944, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gpt-5.3-codex": { - "spend": 0.0073632, - "input_cost": 0.0028032, - "output_cost": 0.00456, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gpt-5.5-pro": { - "spend": 0.073632, - "input_cost": 0.028032, - "output_cost": 0.0456, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "claude-opus-5": { - "spend": 0.018844, - "input_cost": 0.009344, - "output_cost": 0.0095, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "claude-sonnet-5": { - "spend": 0.0113064, - "input_cost": 0.0056064, - "output_cost": 0.0057, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "claude-haiku-4-5": { - "spend": 0.0037688, - "input_cost": 0.0018688, - "output_cost": 0.0019, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.0207284, - "input_cost": 0.0102784, - "output_cost": 0.01045, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01243704, - "input_cost": 0.00616704, - "output_cost": 0.00627, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.0082976, - "input_cost": 0.0037376, - "output_cost": 0.00456, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.0020744, - "input_cost": 0.0009344, - "output_cost": 0.00114, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gemini-3.1-pro": { - "spend": 0.00871248, - "input_cost": 0.00392448, - "output_cost": 0.004788, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "gemini-3.8-flash": { - "spend": 0.002157376, - "input_cost": 0.000971776, - "output_cost": 0.0011856, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.00207128, - "input_cost": 0.00112128, - "output_cost": 0.00095, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.00304992, - "input_cost": 0.00168192, - "output_cost": 0.001368, - "prompt_tokens": 12928, - "completion_tokens": 380 - } - } - }, - { - "name": "cache_write_5m", - "family": "pricing", - "usage": { - "fresh_input_tokens": 512, - "cache_write_5m_tokens": 9216, - "output_tokens": 350 - }, - "owns": [ - "cache_creation_input_token_cost" - ], - "fallback_for": [], - "expected": { - "claude-opus-5": { - "spend": 0.06891, - "input_cost": 0.06016, - "output_cost": 0.00875, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "claude-sonnet-5": { - "spend": 0.041346, - "input_cost": 0.036096, - "output_cost": 0.00525, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "claude-haiku-4-5": { - "spend": 0.013782, - "input_cost": 0.012032, - "output_cost": 0.00175, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.075801, - "input_cost": 0.066176, - "output_cost": 0.009625, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.0454806, - "input_cost": 0.0397056, - "output_cost": 0.005775, - "prompt_tokens": 9728, - "completion_tokens": 350 - } - } - }, - { - "name": "cache_write_1h", - "family": "pricing", - "usage": { - "fresh_input_tokens": 512, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 7168, - "output_tokens": 350 - }, - "owns": [ - "cache_creation_input_token_cost_above_1hr" - ], - "fallback_for": [], - "expected": { - "claude-opus-5": { - "spend": 0.09579, - "input_cost": 0.08704, - "output_cost": 0.00875, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "claude-sonnet-5": { - "spend": 0.057474, - "input_cost": 0.052224, - "output_cost": 0.00525, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "claude-haiku-4-5": { - "spend": 0.019158, - "input_cost": 0.017408, - "output_cost": 0.00175, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.105369, - "input_cost": 0.095744, - "output_cost": 0.009625, - "prompt_tokens": 9728, - "completion_tokens": 350 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.0632214, - "input_cost": 0.0574464, - "output_cost": 0.005775, - "prompt_tokens": 9728, - "completion_tokens": 350 - } - } - }, - { - "name": "audio_input", - "family": "pricing", - "usage": { - "fresh_input_tokens": 96, - "audio_input_tokens": 1450, - "output_tokens": 210 - }, - "owns": [ - "input_cost_per_audio_token" - ], - "fallback_for": [], - "audio_input": true, - "expected": { - "gpt-5.6": { - "spend": 0.061108, - "input_cost": 0.058168, - "output_cost": 0.00294, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "gpt-5.4-mini": { - "spend": 0.0151216, - "input_cost": 0.0145336, - "output_cost": 0.000588, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "azure/gpt-5.6": { - "spend": 0.0626468, - "input_cost": 0.0596228, - "output_cost": 0.003024, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "azure/gpt-5.4-mini": { - "spend": 0.01586436, - "input_cost": 0.01525956, - "output_cost": 0.0006048, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.006482, - "input_cost": 0.003962, - "output_cost": 0.00252, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.002128, - "input_cost": 0.001498, - "output_cost": 0.00063, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "gemini-3.1-pro": { - "spend": 0.0067626, - "input_cost": 0.0041166, - "output_cost": 0.002646, - "prompt_tokens": 1546, - "completion_tokens": 210 - }, - "gemini-3.8-flash": { - "spend": 0.00221312, - "input_cost": 0.00155792, - "output_cost": 0.0006552, - "prompt_tokens": 1546, - "completion_tokens": 210 - } - } - }, - { - "name": "audio_output", - "family": "pricing", - "usage": { - "fresh_input_tokens": 220, - "output_tokens": 180, - "audio_output_tokens": 1120 - }, - "owns": [ - "output_cost_per_audio_token" - ], - "fallback_for": [], - "audio_output": true, - "expected": { - "gpt-5.6": { - "spend": 0.092505, - "input_cost": 0.000385, - "output_cost": 0.09212, - "prompt_tokens": 220, - "completion_tokens": 1300 - }, - "gpt-5.4-mini": { - "spend": 0.022981, - "input_cost": 7.7e-05, - "output_cost": 0.022904, - "prompt_tokens": 220, - "completion_tokens": 1300 - }, - "azure/gpt-5.6": { - "spend": 0.094828, - "input_cost": 0.000396, - "output_cost": 0.094432, - "prompt_tokens": 220, - "completion_tokens": 1300 - }, - "azure/gpt-5.4-mini": { - "spend": 0.0241176, - "input_cost": 7.92e-05, - "output_cost": 0.0240384, - "prompt_tokens": 220, - "completion_tokens": 1300 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.00737, - "input_cost": 0.00011, - "output_cost": 0.00726, - "prompt_tokens": 220, - "completion_tokens": 1300 - }, - "gemini-3.8-flash": { - "spend": 0.0076648, - "input_cost": 0.0001144, - "output_cost": 0.0075504, - "prompt_tokens": 220, - "completion_tokens": 1300 - } - } - }, - { - "name": "image_input", - "family": "pricing", - "usage": { - "fresh_input_tokens": 310, - "image_input_tokens": 1806, - "output_tokens": 240 - }, - "owns": [ - "input_cost_per_image_token" - ], - "fallback_for": [], - "image_input": true, - "expected": { - "gemini/gemini-3.1-pro": { - "spend": 0.0074732, - "input_cost": 0.0045932, - "output_cost": 0.00288, - "prompt_tokens": 2116, - "completion_tokens": 240 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.0018683, - "input_cost": 0.0011483, - "output_cost": 0.00072, - "prompt_tokens": 2116, - "completion_tokens": 240 - }, - "gemini-3.1-pro": { - "spend": 0.0078288, - "input_cost": 0.0048048, - "output_cost": 0.003024, - "prompt_tokens": 2116, - "completion_tokens": 240 - } - } - }, - { - "name": "video_input", - "family": "pricing", - "usage": { - "fresh_input_tokens": 140, - "video_input_tokens": 7920, - "output_tokens": 300 - }, - "owns": [ - "input_cost_per_video_token" - ], - "fallback_for": [], - "video_input": true, - "expected": { - "gemini/gemini-3.1-pro": { - "spend": 0.022888, - "input_cost": 0.019288, - "output_cost": 0.0036, - "prompt_tokens": 8060, - "completion_tokens": 300 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.005722, - "input_cost": 0.004822, - "output_cost": 0.0009, - "prompt_tokens": 8060, - "completion_tokens": 300 - }, - "gemini-3.8-flash": { - "spend": 0.0059192, - "input_cost": 0.0049832, - "output_cost": 0.000936, - "prompt_tokens": 8060, - "completion_tokens": 300 - } - } - }, - { - "name": "reasoning", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1240, - "output_tokens": 560, - "reasoning_tokens": 3480 - }, - "owns": [ - "output_cost_per_reasoning_token" - ], - "fallback_for": [], - "reasoning": true, - "expected": { - "gpt-5.6": { - "spend": 0.06569, - "input_cost": 0.00217, - "output_cost": 0.06352, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "gpt-5.4-mini": { - "spend": 0.013138, - "input_cost": 0.000434, - "output_cost": 0.012704, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "azure/gpt-5.6": { - "spend": 0.067716, - "input_cost": 0.002232, - "output_cost": 0.065484, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "azure/gpt-5.4-mini": { - "spend": 0.0135432, - "input_cost": 0.0004464, - "output_cost": 0.0130968, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "gpt-5.3-codex": { - "spend": 0.05382, - "input_cost": 0.00186, - "output_cost": 0.05196, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "gpt-5.5-pro": { - "spend": 0.5382, - "input_cost": 0.0186, - "output_cost": 0.5196, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.05444, - "input_cost": 0.00248, - "output_cost": 0.05196, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.01448, - "input_cost": 0.00062, - "output_cost": 0.01386, - "prompt_tokens": 1240, - "completion_tokens": 4040 - }, - "gemini-3.1-pro": { - "spend": 0.05664, - "input_cost": 0.002604, - "output_cost": 0.054036, - "prompt_tokens": 1240, - "completion_tokens": 4040 - } - } - }, - { - "name": "tiered_input_above_200k", - "family": "pricing", - "usage": { - "fresh_input_tokens": 204800, - "output_tokens": 620 - }, - "owns": [ - "input_cost_per_token_above_200k_tokens", - "output_cost_per_token_above_200k_tokens" - ], - "fallback_for": [], - "expected": { - "claude-opus-5": { - "spend": 2.07125, - "input_cost": 2.048, - "output_cost": 0.02325, - "prompt_tokens": 204800, - "completion_tokens": 620 - }, - "claude-sonnet-5": { - "spend": 1.24275, - "input_cost": 1.2288, - "output_cost": 0.01395, - "prompt_tokens": 204800, - "completion_tokens": 620 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 2.278375, - "input_cost": 2.2528, - "output_cost": 0.025575, - "prompt_tokens": 204800, - "completion_tokens": 620 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.83036, - "input_cost": 0.8192, - "output_cost": 0.01116, - "prompt_tokens": 204800, - "completion_tokens": 620 - }, - "gemini-3.1-pro": { - "spend": 0.871878, - "input_cost": 0.86016, - "output_cost": 0.011718, - "prompt_tokens": 204800, - "completion_tokens": 620 - } - } - }, - { - "name": "tiered_cache_read_above_200k", - "family": "pricing", - "usage": { - "fresh_input_tokens": 4096, - "cache_read_tokens": 201728, - "output_tokens": 480 - }, - "owns": [ - "cache_read_input_token_cost_above_200k_tokens" - ], - "fallback_for": [], - "expected": { - "claude-opus-5": { - "spend": 0.260688, - "input_cost": 0.242688, - "output_cost": 0.018, - "prompt_tokens": 205824, - "completion_tokens": 480 - }, - "claude-sonnet-5": { - "spend": 0.1564128, - "input_cost": 0.1456128, - "output_cost": 0.0108, - "prompt_tokens": 205824, - "completion_tokens": 480 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.2867568, - "input_cost": 0.2669568, - "output_cost": 0.0198, - "prompt_tokens": 205824, - "completion_tokens": 480 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.1057152, - "input_cost": 0.0970752, - "output_cost": 0.00864, - "prompt_tokens": 205824, - "completion_tokens": 480 - }, - "gemini-3.1-pro": { - "spend": 0.11100096, - "input_cost": 0.10192896, - "output_cost": 0.009072, - "prompt_tokens": 205824, - "completion_tokens": 480 - } - } - }, - { - "name": "tiered_cache_write_above_200k", - "family": "pricing", - "usage": { - "fresh_input_tokens": 4096, - "cache_write_5m_tokens": 200704, - "output_tokens": 480 - }, - "owns": [ - "cache_creation_input_token_cost_above_200k_tokens" - ], - "fallback_for": [], - "expected": { - "claude-opus-5": { - "spend": 2.56776, - "input_cost": 2.54976, - "output_cost": 0.018, - "prompt_tokens": 204800, - "completion_tokens": 480 - }, - "claude-sonnet-5": { - "spend": 1.540656, - "input_cost": 1.529856, - "output_cost": 0.0108, - "prompt_tokens": 204800, - "completion_tokens": 480 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 2.824536, - "input_cost": 2.804736, - "output_cost": 0.0198, - "prompt_tokens": 204800, - "completion_tokens": 480 - } - } - }, - { - "name": "service_tier_flex", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "owns": [ - "input_cost_per_token_flex", - "output_cost_per_token_flex" - ], - "fallback_for": [], - "service_tier": "flex", - "expected": { - "gpt-5.6": { - "spend": 0.004494, - "input_cost": 0.00161, - "output_cost": 0.002884, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0008988, - "input_cost": 0.000322, - "output_cost": 0.0005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0046224, - "input_cost": 0.001656, - "output_cost": 0.0029664, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00092448, - "input_cost": 0.0003312, - "output_cost": 0.00059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.003852, - "input_cost": 0.00138, - "output_cost": 0.002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.03852, - "input_cost": 0.0138, - "output_cost": 0.02472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.010725, - "input_cost": 0.00506, - "output_cost": 0.005665, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.006435, - "input_cost": 0.003036, - "output_cost": 0.003399, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.004312, - "input_cost": 0.00184, - "output_cost": 0.002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.001078, - "input_cost": 0.00046, - "output_cost": 0.000618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.0045276, - "input_cost": 0.001932, - "output_cost": 0.0025956, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.00112112, - "input_cost": 0.0004784, - "output_cost": 0.00064272, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "service_tier_priority", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "owns": [ - "input_cost_per_token_priority", - "output_cost_per_token_priority" - ], - "fallback_for": [], - "service_tier": "priority", - "expected": { - "gpt-5.6": { - "spend": 0.017976, - "input_cost": 0.00644, - "output_cost": 0.011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0035952, - "input_cost": 0.001288, - "output_cost": 0.0023072, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0184896, - "input_cost": 0.006624, - "output_cost": 0.0118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00369792, - "input_cost": 0.0013248, - "output_cost": 0.00237312, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.015408, - "input_cost": 0.00552, - "output_cost": 0.009888, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.15408, - "input_cost": 0.0552, - "output_cost": 0.09888, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.024375, - "input_cost": 0.0115, - "output_cost": 0.012875, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.014625, - "input_cost": 0.0069, - "output_cost": 0.007725, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.004875, - "input_cost": 0.0023, - "output_cost": 0.002575, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.0268125, - "input_cost": 0.01265, - "output_cost": 0.0141625, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.0160875, - "input_cost": 0.00759, - "output_cost": 0.0084975, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.01078, - "input_cost": 0.0046, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.002695, - "input_cost": 0.00115, - "output_cost": 0.001545, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.011319, - "input_cost": 0.00483, - "output_cost": 0.006489, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.0028028, - "input_cost": 0.001196, - "output_cost": 0.0016068, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "anthropic_fast_mode", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "owns": [ - "provider_specific_entry.fast" - ], - "fallback_for": [], - "speed": "fast", - "expected": { - "claude-opus-5": { - "spend": 0.117, - "input_cost": 0.0552, - "output_cost": 0.0618, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "anthropic_us_inference", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "owns": [ - "provider_specific_entry.us" - ], - "fallback_for": [], - "inference_geo": "us", - "expected": { - "claude-opus-5": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.00429, - "input_cost": 0.002024, - "output_cost": 0.002266, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "web_search_medium", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "web_search_calls": 3 - }, - "owns": [ - "search_context_cost_per_query.search_context_size_medium", - "web_search_billing_unit" - ], - "fallback_for": [], - "web_search": "medium", - "expected": { - "gpt-5.6": { - "spend": 0.021488, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0142976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0217448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.01434896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.045204, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.11454, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0495, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0417, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0339, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.113624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.1140552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "web_search_low", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "web_search_calls": 1 - }, - "owns": [ - "search_context_cost_per_query.search_context_size_low" - ], - "fallback_for": [], - "web_search": "low", - "expected": { - "gpt-5.6": { - "spend": 0.018988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0117976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0192448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.01184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.017704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.08704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "web_search_high", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "web_search_calls": 1 - }, - "owns": [ - "search_context_cost_per_query.search_context_size_high" - ], - "fallback_for": [], - "web_search": "high", - "expected": { - "gpt-5.6": { - "spend": 0.023988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0167976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0242448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.01684896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.022704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.09204, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "web_search_per_prompt", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "web_search_calls": 3 - }, - "owns": [ - "search_context_cost_per_query.search_context_size_medium", - "web_search_billing_unit" - ], - "fallback_for": [], - "web_search": "medium", - "expected": { - "gemini/gemini-3.8-flash": { - "spend": 0.037156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.03724224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "google_maps_grounding", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "google_maps_calls": 1 - }, - "owns": [ - "google_maps_grounding_cost_per_query" - ], - "fallback_for": [], - "google_maps": true, - "expected": { - "gemini/gemini-3.1-pro": { - "spend": 0.033624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.027156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.0340552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.02724224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "file_search", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "file_search_calls": 1 - }, - "owns": [ - "file_search_cost_per_1k_calls" - ], - "fallback_for": [], - "file_search": true, - "expected": { - "gpt-5.3-codex": { - "spend": 0.010204, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07954, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "fallback_cache_read_at_input_rate", - "family": "pricing", - "usage": { - "fresh_input_tokens": 640, - "cache_read_tokens": 12288, - "output_tokens": 380 - }, - "owns": [], - "fallback_for": [ - "cache_read_input_token_cost" - ], - "expected": { - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00347132, - "input_cost": 0.00310272, - "output_cost": 0.0003686, - "prompt_tokens": 12928, - "completion_tokens": 380 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.0021672, - "input_cost": 0.0019392, - "output_cost": 0.000228, - "prompt_tokens": 12928, - "completion_tokens": 380 - } - } - }, - { - "name": "fallback_cache_write_at_input_rate", - "family": "pricing", - "usage": { - "fresh_input_tokens": 512, - "cache_write_5m_tokens": 9216, - "output_tokens": 350 - }, - "owns": [], - "fallback_for": [ - "cache_creation_input_token_cost" - ], - "expected": { - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00267422, - "input_cost": 0.00233472, - "output_cost": 0.0003395, - "prompt_tokens": 9728, - "completion_tokens": 350 - } - } - }, - { - "name": "fallback_reasoning_at_output_rate", - "family": "pricing", - "usage": { - "fresh_input_tokens": 1240, - "output_tokens": 560, - "reasoning_tokens": 3480 - }, - "owns": [], - "fallback_for": [ - "output_cost_per_reasoning_token" - ], - "reasoning": true, - "expected": { - "gemini-3.8-flash": { - "spend": 0.0132496, - "input_cost": 0.0006448, - "output_cost": 0.0126048, - "prompt_tokens": 1240, - "completion_tokens": 4040 - } - } - }, - { - "name": "fallback_image_tokens_at_input_rate", - "family": "pricing", - "usage": { - "fresh_input_tokens": 310, - "image_input_tokens": 1806, - "output_tokens": 240 - }, - "owns": [], - "fallback_for": [ - "input_cost_per_image_token" - ], - "image_input": true, - "expected": { - "gemini-3.8-flash": { - "spend": 0.00184912, - "input_cost": 0.00110032, - "output_cost": 0.0007488, - "prompt_tokens": 2116, - "completion_tokens": 240 - } - } - }, - { - "name": "fallback_video_tokens_at_input_rate", - "family": "pricing", - "usage": { - "fresh_input_tokens": 140, - "video_input_tokens": 7920, - "output_tokens": 300 - }, - "owns": [], - "fallback_for": [ - "input_cost_per_video_token" - ], - "video_input": true, - "expected": { - "gemini-3.1-pro": { - "spend": 0.020706, - "input_cost": 0.016926, - "output_cost": 0.00378, - "prompt_tokens": 8060, - "completion_tokens": 300 - } - } - }, - { - "name": "stream", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "expected": { - "gpt-5.6": { - "spend": 0.008988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0017976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0092448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0195, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0117, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0039, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00084124, - "input_cost": 0.0004416, - "output_cost": 0.00039964, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.008624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.002156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.0090552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.00224224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.002134, - "input_cost": 0.001104, - "output_cost": 0.00103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.0031392, - "input_cost": 0.001656, - "output_cost": 0.0014832, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "stream_no_usage", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "stream_usage": "absent", - "exact_spend": false, - "models": [ - "gpt-5.6", - "gpt-5.4-mini", - "azure/gpt-5.6", - "azure/gpt-5.4-mini", - "gpt-5.3-codex", - "gpt-5.5-pro", - "claude-opus-5", - "claude-sonnet-5", - "claude-haiku-4-5", - "us.anthropic.claude-opus-5-v1:0", - "anthropic.claude-sonnet-5-v1:0", - "meta.llama4-maverick-17b-instruct-v1:0", - "gemini/gemini-3.1-pro", - "gemini/gemini-3.8-flash", - "gemini-3.1-pro", - "gemini-3.8-flash", - "together_ai/moonshotai/Kimi-K3", - "together_ai/zai-org/GLM-5.3", - "fireworks_ai/accounts/fireworks/models/kimi-k3", - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", - "fireworks_ai/accounts/fireworks/models/qwen3p8-max" - ] - }, - { - "name": "stream_no_usage_tool_call", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "stream_usage": "absent", - "tool_call": true, - "exact_spend": false, - "models": [ - "gpt-5.6", - "gpt-5.4-mini", - "azure/gpt-5.6", - "azure/gpt-5.4-mini", - "gpt-5.3-codex", - "gpt-5.5-pro", - "claude-opus-5", - "claude-sonnet-5", - "claude-haiku-4-5", - "us.anthropic.claude-opus-5-v1:0", - "anthropic.claude-sonnet-5-v1:0", - "meta.llama4-maverick-17b-instruct-v1:0", - "gemini/gemini-3.1-pro", - "gemini/gemini-3.8-flash", - "gemini-3.1-pro", - "gemini-3.8-flash", - "together_ai/moonshotai/Kimi-K3", - "together_ai/zai-org/GLM-5.3", - "fireworks_ai/accounts/fireworks/models/kimi-k3", - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", - "fireworks_ai/accounts/fireworks/models/qwen3p8-max" - ] - }, - { - "name": "stream_no_usage_image_input", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "stream_usage": "absent", - "image_input": true, - "exact_spend": false, - "models": [ - "gpt-5.6", - "gpt-5.4-mini", - "azure/gpt-5.6", - "azure/gpt-5.4-mini", - "gpt-5.3-codex", - "gpt-5.5-pro", - "claude-opus-5", - "claude-sonnet-5", - "claude-haiku-4-5", - "us.anthropic.claude-opus-5-v1:0", - "anthropic.claude-sonnet-5-v1:0", - "meta.llama4-maverick-17b-instruct-v1:0", - "gemini/gemini-3.1-pro", - "gemini/gemini-3.8-flash", - "gemini-3.1-pro", - "gemini-3.8-flash", - "together_ai/moonshotai/Kimi-K3", - "together_ai/zai-org/GLM-5.3", - "fireworks_ai/accounts/fireworks/models/kimi-k3", - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", - "fireworks_ai/accounts/fireworks/models/qwen3p8-max" - ] - }, - { - "name": "stream_incomplete", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "terminal": "incomplete", - "expected": { - "gpt-5.3-codex": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "stream_no_usage_incomplete", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "stream_usage": "absent", - "terminal": "incomplete", - "exact_spend": false, - "models": [ - "gpt-5.3-codex", - "gpt-5.5-pro" - ] - }, - { - "name": "stream_unvalidated", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "terminal": "unvalidated", - "expected": { - "gpt-5.3-codex": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "stream_no_usage_unvalidated", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "stream_usage": "absent", - "terminal": "unvalidated", - "exact_spend": false, - "models": [ - "gpt-5.3-codex", - "gpt-5.5-pro" - ] - }, - { - "name": "prompt_blocked", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840 - }, - "terminal": "prompt_blocked", - "expected": { - "gemini/gemini-3.1-pro": { - "spend": 0.00368, - "input_cost": 0.00368, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.00092, - "input_cost": 0.00092, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - }, - "gemini-3.1-pro": { - "spend": 0.003864, - "input_cost": 0.003864, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - }, - "gemini-3.8-flash": { - "spend": 0.0009568, - "input_cost": 0.0009568, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - } - } - }, - { - "name": "stream_prompt_blocked", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840 - }, - "stream": true, - "terminal": "prompt_blocked", - "expected": { - "gemini/gemini-3.1-pro": { - "spend": 0.00368, - "input_cost": 0.00368, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.00092, - "input_cost": 0.00092, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - }, - "gemini-3.1-pro": { - "spend": 0.003864, - "input_cost": 0.003864, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - }, - "gemini-3.8-flash": { - "spend": 0.0009568, - "input_cost": 0.0009568, - "output_cost": 0.0, - "prompt_tokens": 1840, - "completion_tokens": 0 - } - } - }, - { - "name": "response_model_override", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "response_model_override": true, - "expected": { - "gpt-5.6": { - "spend": 0.0017976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.008988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0117, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0039, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0195, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00084124, - "input_cost": 0.0004416, - "output_cost": 0.00039964, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.002156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.008624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.00224224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.0090552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.0031392, - "input_cost": 0.001656, - "output_cost": 0.0014832, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.002134, - "input_cost": 0.001104, - "output_cost": 0.00103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "stream_response_model_override", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "response_model_override": true, - "stream": true, - "expected": { - "gpt-5.6": { - "spend": 0.0017976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.008988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0117, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0039, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0195, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00084124, - "input_cost": 0.0004416, - "output_cost": 0.00039964, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.002156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.008624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.00224224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.0090552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.0031392, - "input_cost": 0.001656, - "output_cost": 0.0014832, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.002134, - "input_cost": 0.001104, - "output_cost": 0.00103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "tool_call", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "tool_call": true, - "expected": { - "gpt-5.6": { - "spend": 0.008988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0017976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0092448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0195, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0117, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0039, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00084124, - "input_cost": 0.0004416, - "output_cost": 0.00039964, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.008624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.002156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.0090552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.00224224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.002134, - "input_cost": 0.001104, - "output_cost": 0.00103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.0031392, - "input_cost": 0.001656, - "output_cost": 0.0014832, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "stream_tool_call", - "family": "transport", - "usage": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "stream": true, - "tool_call": true, - "expected": { - "gpt-5.6": { - "spend": 0.008988, - "input_cost": 0.00322, - "output_cost": 0.005768, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.4-mini": { - "spend": 0.0017976, - "input_cost": 0.000644, - "output_cost": 0.0011536, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.6": { - "spend": 0.0092448, - "input_cost": 0.003312, - "output_cost": 0.0059328, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "azure/gpt-5.4-mini": { - "spend": 0.00184896, - "input_cost": 0.0006624, - "output_cost": 0.00118656, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.3-codex": { - "spend": 0.007704, - "input_cost": 0.00276, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gpt-5.5-pro": { - "spend": 0.07704, - "input_cost": 0.0276, - "output_cost": 0.04944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-opus-5": { - "spend": 0.0195, - "input_cost": 0.0092, - "output_cost": 0.0103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0117, - "input_cost": 0.00552, - "output_cost": 0.00618, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0039, - "input_cost": 0.00184, - "output_cost": 0.00206, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.02145, - "input_cost": 0.01012, - "output_cost": 0.01133, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.01287, - "input_cost": 0.006072, - "output_cost": 0.006798, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00084124, - "input_cost": 0.0004416, - "output_cost": 0.00039964, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.008624, - "input_cost": 0.00368, - "output_cost": 0.004944, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.002156, - "input_cost": 0.00092, - "output_cost": 0.001236, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.1-pro": { - "spend": 0.0090552, - "input_cost": 0.003864, - "output_cost": 0.0051912, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "gemini-3.8-flash": { - "spend": 0.00224224, - "input_cost": 0.0009568, - "output_cost": 0.00128544, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.002134, - "input_cost": 0.001104, - "output_cost": 0.00103, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.0031392, - "input_cost": 0.001656, - "output_cost": 0.0014832, - "prompt_tokens": 1840, - "completion_tokens": 412 - } - } - }, - { - "name": "stream_full_usage", - "family": "transport", - "usage": {}, - "stream": true, - "usage_by_model": { - "gpt-5.6": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330, - "audio_output_tokens": 280 - }, - "gpt-5.4-mini": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330, - "audio_output_tokens": 280 - }, - "azure/gpt-5.6": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330, - "audio_output_tokens": 280 - }, - "azure/gpt-5.4-mini": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330, - "audio_output_tokens": 280 - }, - "gpt-5.3-codex": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900 - }, - "gpt-5.5-pro": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900 - }, - "claude-opus-5": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 1024 - }, - "claude-sonnet-5": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 1024 - }, - "claude-haiku-4-5": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 1024 - }, - "us.anthropic.claude-opus-5-v1:0": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 1024 - }, - "anthropic.claude-sonnet-5-v1:0": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 1024 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "cache_write_5m_tokens": 2048, - "cache_write_1h_tokens": 1024 - }, - "gemini/gemini-3.1-pro": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330 - }, - "gemini/gemini-3.8-flash": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330, - "audio_output_tokens": 280 - }, - "gemini-3.1-pro": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330 - }, - "gemini-3.8-flash": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144, - "reasoning_tokens": 900, - "audio_input_tokens": 330, - "audio_output_tokens": 280 - }, - "together_ai/moonshotai/Kimi-K3": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "fresh_input_tokens": 1840, - "output_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "fresh_input_tokens": 1840, - "output_tokens": 412, - "cache_read_tokens": 6144 - } - }, - "expected": { - "gpt-5.6": { - "spend": 0.0600632, - "input_cost": 0.0174952, - "output_cost": 0.042568, - "prompt_tokens": 8314, - "completion_tokens": 1592 - }, - "gpt-5.4-mini": { - "spend": 0.01379264, - "input_cost": 0.00415904, - "output_cost": 0.0096336, - "prompt_tokens": 8314, - "completion_tokens": 1592 - }, - "azure/gpt-5.6": { - "spend": 0.06169072, - "input_cost": 0.01794792, - "output_cost": 0.0437428, - "prompt_tokens": 8314, - "completion_tokens": 1592 - }, - "azure/gpt-5.4-mini": { - "spend": 0.014385144, - "input_cost": 0.004348584, - "output_cost": 0.01003656, - "prompt_tokens": 8314, - "completion_tokens": 1592 - }, - "gpt-5.3-codex": { - "spend": 0.0203256, - "input_cost": 0.0036816, - "output_cost": 0.016644, - "prompt_tokens": 7984, - "completion_tokens": 1312 - }, - "gpt-5.5-pro": { - "spend": 0.203256, - "input_cost": 0.036816, - "output_cost": 0.16644, - "prompt_tokens": 7984, - "completion_tokens": 1312 - }, - "claude-opus-5": { - "spend": 0.045612, - "input_cost": 0.035312, - "output_cost": 0.0103, - "prompt_tokens": 11056, - "completion_tokens": 412 - }, - "claude-sonnet-5": { - "spend": 0.0273672, - "input_cost": 0.0211872, - "output_cost": 0.00618, - "prompt_tokens": 11056, - "completion_tokens": 412 - }, - "claude-haiku-4-5": { - "spend": 0.0091224, - "input_cost": 0.0070624, - "output_cost": 0.00206, - "prompt_tokens": 11056, - "completion_tokens": 412 - }, - "us.anthropic.claude-opus-5-v1:0": { - "spend": 0.0501732, - "input_cost": 0.0388432, - "output_cost": 0.01133, - "prompt_tokens": 11056, - "completion_tokens": 412 - }, - "anthropic.claude-sonnet-5-v1:0": { - "spend": 0.03010392, - "input_cost": 0.02330592, - "output_cost": 0.006798, - "prompt_tokens": 11056, - "completion_tokens": 412 - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "spend": 0.00305308, - "input_cost": 0.00265344, - "output_cost": 0.00039964, - "prompt_tokens": 11056, - "completion_tokens": 412 - }, - "gemini/gemini-3.1-pro": { - "spend": 0.0224108, - "input_cost": 0.0057668, - "output_cost": 0.016644, - "prompt_tokens": 8314, - "completion_tokens": 1312 - }, - "gemini/gemini-3.8-flash": { - "spend": 0.0076232, - "input_cost": 0.0015572, - "output_cost": 0.006066, - "prompt_tokens": 8314, - "completion_tokens": 1592 - }, - "gemini-3.1-pro": { - "spend": 0.02338644, - "input_cost": 0.00604524, - "output_cost": 0.0173412, - "prompt_tokens": 8314, - "completion_tokens": 1312 - }, - "gemini-3.8-flash": { - "spend": 0.007460128, - "input_cost": 0.001619488, - "output_cost": 0.00584064, - "prompt_tokens": 8314, - "completion_tokens": 1592 - }, - "together_ai/moonshotai/Kimi-K3": { - "spend": 0.0035374, - "input_cost": 0.002116, - "output_cost": 0.0014214, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "together_ai/zai-org/GLM-5.3": { - "spend": 0.0019184, - "input_cost": 0.001012, - "output_cost": 0.0009064, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "spend": 0.00250264, - "input_cost": 0.00147264, - "output_cost": 0.00103, - "prompt_tokens": 7984, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "spend": 0.0005232, - "input_cost": 0.000276, - "output_cost": 0.0002472, - "prompt_tokens": 1840, - "completion_tokens": 412 - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "spend": 0.00369216, - "input_cost": 0.00220896, - "output_cost": 0.0014832, - "prompt_tokens": 7984, - "completion_tokens": 412 - } - } - } - ] -} diff --git a/tests/integration/cost_calculation/conftest.py b/tests/integration/cost_calculation/conftest.py index 9229bb47817..f1b8901d626 100644 --- a/tests/integration/cost_calculation/conftest.py +++ b/tests/integration/cost_calculation/conftest.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, ConfigDict from integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value from integration._support.database import read_rows from integration._support.upstream import delete_scenario, register_scenario -from integration.cost_calculation.cost_matrix import Case, FrontierModel +from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase class CostBreakdown(BaseModel): @@ -118,25 +118,23 @@ def _vertex_service_account_json(url: str) -> str: def register_scenario_deployment( scenario: Scenario, - model: FrontierModel, - case: Case, + case: CostTrackingTestCase, marker: str, + key: str, ) -> str: control_url: Final = os.environ["INTEGRATION_UPSTREAM_URL"].rstrip("/") - sidecar_scenario: Final = case.scenario( - scenario_id=f"sc-{marker}", model=model, text=f"scripted answer {marker}" - ) - handle: Final = register_scenario(sidecar_scenario) + run_marker: Final = sha256(key.encode()).hexdigest()[:12] + handle: Final = register_scenario(f"sc-{marker}-{run_marker}", case.response) scenario.cleanups.callback(delete_scenario, handle) - model_name: Final = f"{model.model_name}-{marker}" + model_name: Final = f"cost-{marker}-{run_marker}" parameters: Final = { - "model": model.litellm_model, - "api_key": model.api_key, + "model": case.litellm_model, + "api_key": case.api_key, "api_base": handle.api_base(), - **model.litellm_params, + **case.litellm_params, **( {"vertex_credentials": _vertex_service_account_json(control_url)} - if model.llm_provider == "vertex_ai" + if case.rates.litellm_provider == "vertex_ai-language-models" else {} ), } @@ -145,7 +143,11 @@ def register_scenario_deployment( JSON_OBJECT.validate_python({ "model_name": model_name, "litellm_params": parameters, - "model_info": {"base_model": model.base_model}, + "model_info": ( + {"base_model": case.base_model} + if case.base_model is not None + else {} + ), }), ) identity: Final = string_value(object_value(created["model_info"])["id"]) diff --git a/tests/integration/cost_calculation/cost_map.json b/tests/integration/cost_calculation/cost_map.json deleted file mode 100644 index 117e9b33636..00000000000 --- a/tests/integration/cost_calculation/cost_map.json +++ /dev/null @@ -1,411 +0,0 @@ -{ - "gpt-5.6": { - "cache_read_input_token_cost": 1.75e-07, - "input_cost_per_audio_token": 4e-05, - "input_cost_per_token": 1.75e-06, - "input_cost_per_token_flex": 8.75e-07, - "input_cost_per_token_priority": 3.5e-06, - "litellm_provider": "openai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_audio_token": 8e-05, - "output_cost_per_reasoning_token": 1.6e-05, - "output_cost_per_token": 1.4e-05, - "output_cost_per_token_flex": 7e-06, - "output_cost_per_token_priority": 2.8e-05, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.0125, - "search_context_size_high": 0.015 - }, - "supports_function_calling": true - }, - "gpt-5.4-mini": { - "cache_read_input_token_cost": 3.5e-08, - "input_cost_per_audio_token": 1e-05, - "input_cost_per_token": 3.5e-07, - "input_cost_per_token_flex": 1.75e-07, - "input_cost_per_token_priority": 7e-07, - "litellm_provider": "openai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_reasoning_token": 3.2e-06, - "output_cost_per_token": 2.8e-06, - "output_cost_per_token_flex": 1.4e-06, - "output_cost_per_token_priority": 5.6e-06, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.0125, - "search_context_size_high": 0.015 - }, - "supports_function_calling": true - }, - "azure/gpt-5.6": { - "cache_read_input_token_cost": 1.8e-07, - "input_cost_per_audio_token": 4.1e-05, - "input_cost_per_token": 1.8e-06, - "input_cost_per_token_flex": 9e-07, - "input_cost_per_token_priority": 3.6e-06, - "litellm_provider": "azure", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_audio_token": 8.2e-05, - "output_cost_per_reasoning_token": 1.65e-05, - "output_cost_per_token": 1.44e-05, - "output_cost_per_token_flex": 7.2e-06, - "output_cost_per_token_priority": 2.88e-05, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.0125, - "search_context_size_high": 0.015 - }, - "supports_function_calling": true - }, - "azure/gpt-5.4-mini": { - "cache_read_input_token_cost": 3.6e-08, - "input_cost_per_audio_token": 1.05e-05, - "input_cost_per_token": 3.6e-07, - "input_cost_per_token_flex": 1.8e-07, - "input_cost_per_token_priority": 7.2e-07, - "litellm_provider": "azure", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_audio_token": 2.1e-05, - "output_cost_per_reasoning_token": 3.3e-06, - "output_cost_per_token": 2.88e-06, - "output_cost_per_token_flex": 1.44e-06, - "output_cost_per_token_priority": 5.76e-06, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.0125, - "search_context_size_high": 0.015 - }, - "supports_function_calling": true - }, - "gpt-5.3-codex": { - "cache_read_input_token_cost": 1.5e-07, - "file_search_cost_per_1k_calls": 0.0025, - "input_cost_per_token": 1.5e-06, - "input_cost_per_token_flex": 7.5e-07, - "input_cost_per_token_priority": 3e-06, - "litellm_provider": "openai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "responses", - "output_cost_per_reasoning_token": 1.3e-05, - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_flex": 6e-06, - "output_cost_per_token_priority": 2.4e-05, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.0125, - "search_context_size_high": 0.015 - }, - "supports_function_calling": true - }, - "gpt-5.5-pro": { - "cache_read_input_token_cost": 1.5e-06, - "file_search_cost_per_1k_calls": 0.0025, - "input_cost_per_token": 1.5e-05, - "input_cost_per_token_flex": 7.5e-06, - "input_cost_per_token_priority": 3e-05, - "litellm_provider": "openai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "responses", - "output_cost_per_reasoning_token": 0.00013, - "output_cost_per_token": 0.00012, - "output_cost_per_token_flex": 6e-05, - "output_cost_per_token_priority": 0.00024, - "search_context_cost_per_query": { - "search_context_size_low": 0.01, - "search_context_size_medium": 0.0125, - "search_context_size_high": 0.015 - }, - "supports_function_calling": true - }, - "claude-opus-5": { - "cache_creation_input_token_cost": 6.25e-06, - "cache_creation_input_token_cost_above_1hr": 1e-05, - "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, - "cache_read_input_token_cost": 5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1e-06, - "input_cost_per_token": 5e-06, - "input_cost_per_token_above_200k_tokens": 1e-05, - "input_cost_per_token_priority": 6.25e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 2.5e-05, - "output_cost_per_token_above_200k_tokens": 3.75e-05, - "output_cost_per_token_priority": 3.125e-05, - "provider_specific_entry": { - "fast": 6.0, - "us": 1.1 - }, - "search_context_cost_per_query": { - "search_context_size_medium": 0.01 - }, - "supports_function_calling": true - }, - "claude-sonnet-5": { - "cache_creation_input_token_cost": 3.75e-06, - "cache_creation_input_token_cost_above_1hr": 6e-06, - "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost": 3e-07, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "input_cost_per_token": 3e-06, - "input_cost_per_token_above_200k_tokens": 6e-06, - "input_cost_per_token_priority": 3.75e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_cost_per_token_above_200k_tokens": 2.25e-05, - "output_cost_per_token_priority": 1.875e-05, - "provider_specific_entry": { - "us": 1.1 - }, - "search_context_cost_per_query": { - "search_context_size_medium": 0.01 - }, - "supports_function_calling": true - }, - "claude-haiku-4-5": { - "cache_creation_input_token_cost": 1.25e-06, - "cache_creation_input_token_cost_above_1hr": 2e-06, - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, - "input_cost_per_token_priority": 1.25e-06, - "litellm_provider": "anthropic", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 5e-06, - "output_cost_per_token_priority": 6.25e-06, - "provider_specific_entry": { - "us": 1.1 - }, - "search_context_cost_per_query": { - "search_context_size_medium": 0.01 - }, - "supports_function_calling": true - }, - "us.anthropic.claude-opus-5-v1:0": { - "cache_creation_input_token_cost": 6.875e-06, - "cache_creation_input_token_cost_above_1hr": 1.1e-05, - "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, - "cache_read_input_token_cost": 5.5e-07, - "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, - "input_cost_per_token": 5.5e-06, - "input_cost_per_token_above_200k_tokens": 1.1e-05, - "input_cost_per_token_flex": 2.75e-06, - "input_cost_per_token_priority": 6.875e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 2.75e-05, - "output_cost_per_token_above_200k_tokens": 4.125e-05, - "output_cost_per_token_flex": 1.375e-05, - "output_cost_per_token_priority": 3.4375e-05, - "supports_function_calling": true - }, - "anthropic.claude-sonnet-5-v1:0": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_creation_input_token_cost_above_1hr": 6.6e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, - "input_cost_per_token_flex": 1.65e-06, - "input_cost_per_token_priority": 4.125e-06, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1.65e-05, - "output_cost_per_token_flex": 8.25e-06, - "output_cost_per_token_priority": 2.0625e-05, - "supports_function_calling": true - }, - "meta.llama4-maverick-17b-instruct-v1:0": { - "input_cost_per_token": 2.4e-07, - "litellm_provider": "bedrock_converse", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 9.7e-07, - "supports_function_calling": true - }, - "gemini/gemini-3.1-pro": { - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_200k_tokens": 4e-07, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token": 2.6e-06, - "input_cost_per_image_token": 2.2e-06, - "input_cost_per_token": 2e-06, - "input_cost_per_token_above_200k_tokens": 4e-06, - "input_cost_per_token_flex": 1e-06, - "input_cost_per_token_priority": 2.5e-06, - "input_cost_per_video_token": 2.4e-06, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 1.3e-05, - "output_cost_per_token": 1.2e-05, - "output_cost_per_token_above_200k_tokens": 1.8e-05, - "output_cost_per_token_flex": 6e-06, - "output_cost_per_token_priority": 1.5e-05, - "search_context_cost_per_query": { - "search_context_size_medium": 0.035 - }, - "supports_function_calling": true, - "web_search_billing_unit": "per_query" - }, - "gemini/gemini-3.8-flash": { - "cache_read_input_token_cost": 5e-08, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token": 1e-06, - "input_cost_per_image_token": 5.5e-07, - "input_cost_per_token": 5e-07, - "input_cost_per_token_flex": 2.5e-07, - "input_cost_per_token_priority": 6.25e-07, - "input_cost_per_video_token": 6e-07, - "litellm_provider": "gemini", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_audio_token": 6e-06, - "output_cost_per_reasoning_token": 3.5e-06, - "output_cost_per_token": 3e-06, - "output_cost_per_token_flex": 1.5e-06, - "output_cost_per_token_priority": 3.75e-06, - "search_context_cost_per_query": { - "search_context_size_medium": 0.035 - }, - "supports_function_calling": true, - "web_search_billing_unit": "per_prompt" - }, - "gemini-3.1-pro": { - "cache_read_input_token_cost": 2.1e-07, - "cache_read_input_token_cost_above_200k_tokens": 4.2e-07, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token": 2.7e-06, - "input_cost_per_image_token": 2.3e-06, - "input_cost_per_token": 2.1e-06, - "input_cost_per_token_above_200k_tokens": 4.2e-06, - "input_cost_per_token_flex": 1.05e-06, - "input_cost_per_token_priority": 2.625e-06, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_reasoning_token": 1.35e-05, - "output_cost_per_token": 1.26e-05, - "output_cost_per_token_above_200k_tokens": 1.89e-05, - "output_cost_per_token_flex": 6.3e-06, - "output_cost_per_token_priority": 1.575e-05, - "search_context_cost_per_query": { - "search_context_size_medium": 0.035 - }, - "supports_function_calling": true, - "web_search_billing_unit": "per_query" - }, - "gemini-3.8-flash": { - "cache_read_input_token_cost": 5.2e-08, - "google_maps_grounding_cost_per_query": 0.025, - "input_cost_per_audio_token": 1.04e-06, - "input_cost_per_token": 5.2e-07, - "input_cost_per_token_flex": 2.6e-07, - "input_cost_per_token_priority": 6.5e-07, - "input_cost_per_video_token": 6.2e-07, - "litellm_provider": "vertex_ai-language-models", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_audio_token": 6.24e-06, - "output_cost_per_token": 3.12e-06, - "output_cost_per_token_flex": 1.56e-06, - "output_cost_per_token_priority": 3.9e-06, - "search_context_cost_per_query": { - "search_context_size_medium": 0.035 - }, - "supports_function_calling": true, - "web_search_billing_unit": "per_prompt" - }, - "together_ai/moonshotai/Kimi-K3": { - "input_cost_per_token": 1.15e-06, - "litellm_provider": "together_ai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 3.45e-06, - "supports_function_calling": true - }, - "together_ai/zai-org/GLM-5.3": { - "input_cost_per_token": 5.5e-07, - "litellm_provider": "together_ai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 2.2e-06, - "supports_function_calling": true - }, - "fireworks_ai/accounts/fireworks/models/kimi-k3": { - "cache_read_input_token_cost": 6e-08, - "input_cost_per_token": 6e-07, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 2.5e-06, - "supports_function_calling": true - }, - "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 6e-07, - "supports_function_calling": true - }, - "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { - "cache_read_input_token_cost": 9e-08, - "input_cost_per_token": 9e-07, - "litellm_provider": "fireworks_ai", - "max_input_tokens": 400000, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 3.6e-06, - "supports_function_calling": true - } -} diff --git a/tests/integration/cost_calculation/cost_matrix.py b/tests/integration/cost_calculation/cost_matrix.py deleted file mode 100644 index b261deb68b2..00000000000 --- a/tests/integration/cost_calculation/cost_matrix.py +++ /dev/null @@ -1,658 +0,0 @@ -"""The cost-calculation matrix: the model set derived from the test cost map, -the request/response cases from ``cases.json``, and the loaders both use. - -Two data files drive the suite; nothing in Python lists models or cases: -- ``tests/integration/cost_calculation/cost_map.json`` is the proxy's ENTIRE model cost map - (LITELLM_MODEL_COST_MAP_URL); every entry becomes a deployment under test. -- ``tests/integration/cost_calculation/cases.json`` is the case list plus the reviewed - goldens: each exact-spend case carries an ``expected`` cell per map key it - runs against, each recount case carries its ``models`` list, so matrix - membership and expected values are literal data read side by side. -""" - -from __future__ import annotations - -import base64 -import io -import json -import math -import random -import struct -import wave -import zlib -from collections.abc import Mapping -from dataclasses import dataclass -from pathlib import Path -from types import MappingProxyType -from typing import Final, Literal - -from litellm import get_llm_provider -from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIConfig -from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig -from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig -from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig -from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter -from integration._support.scripted_shapes import ( - Scenario, - Shape, - ScriptedOutput, - ScriptedToolCall, - ScriptedUsage, -) - -COST_MAP_PATH: Final = Path(__file__).resolve().parent / "cost_map.json" -CASES_PATH: Final = Path(__file__).resolve().parent / "cases.json" - -class SearchContextCostPerQuery(BaseModel): - model_config = ConfigDict(frozen=True) - - search_context_size_low: float | None = None - search_context_size_medium: float | None = None - search_context_size_high: float | None = None - - -class ProviderSpecificEntry(BaseModel): - """Provider-specific key rates, keyed by the named suffix litellm looks up - (``fast`` for Anthropic fast mode, ``us`` for US inference geography).""" - - model_config = ConfigDict(frozen=True) - - fast: float | None = None - us: float | None = None - - -class CostMapEntry(BaseModel): - """The pricing fields of a cost-map entry the matrix reads. Shaped like a - ``model_prices_and_context_window.json`` entry; the file is test-owned so - undeclared keys are forbidden rather than ignored.""" - - model_config = ConfigDict(frozen=True, extra="forbid") - - litellm_provider: str - mode: str - max_tokens: int | None = None - max_input_tokens: int | None = None - max_output_tokens: int | None = None - supports_function_calling: bool | None = None - input_cost_per_token: float | None = None - output_cost_per_token: float | None = None - cache_read_input_token_cost: float | None = None - cache_creation_input_token_cost: float | None = None - cache_creation_input_token_cost_above_1hr: float | None = None - cache_read_input_token_cost_above_200k_tokens: float | None = None - cache_creation_input_token_cost_above_200k_tokens: float | None = None - output_cost_per_reasoning_token: float | None = None - input_cost_per_audio_token: float | None = None - output_cost_per_audio_token: float | None = None - input_cost_per_image_token: float | None = None - input_cost_per_video_token: float | None = None - input_cost_per_token_above_200k_tokens: float | None = None - output_cost_per_token_above_200k_tokens: float | None = None - input_cost_per_token_flex: float | None = None - output_cost_per_token_flex: float | None = None - input_cost_per_token_priority: float | None = None - output_cost_per_token_priority: float | None = None - search_context_cost_per_query: SearchContextCostPerQuery | None = None - web_search_billing_unit: str | None = None - google_maps_grounding_cost_per_query: float | None = None - file_search_cost_per_1k_calls: float | None = None - provider_specific_entry: ProviderSpecificEntry | None = None - - -_METADATA_FIELDS: Final = frozenset( - { - "litellm_provider", - "mode", - "max_tokens", - "max_input_tokens", - "max_output_tokens", - "supports_function_calling", - } -) -_CONTAINER_FIELDS: Final = frozenset({"search_context_cost_per_query", "provider_specific_entry"}) - - -def _submodel_rate_keys( - field: str, sub: SearchContextCostPerQuery | ProviderSpecificEntry | None -) -> tuple[str, ...]: - if sub is None: - return () - return tuple( - f"{field}.{name}" - for name in type(sub).model_fields - if getattr(sub, name) is not None - ) - - -def _entry_rate_keys(entry: CostMapEntry) -> frozenset[str]: - """Every cost key an entry carries, with container subfields expanded to - dotted names (``search_context_cost_per_query.search_context_size_low``). - ``web_search_billing_unit`` counts as a rate key whenever present, - for both ``per_query`` and ``per_prompt`` values.""" - plain: Final = frozenset( - name - for name in CostMapEntry.model_fields - if name not in _METADATA_FIELDS - and name not in _CONTAINER_FIELDS - and getattr(entry, name) is not None - ) - return ( - plain - | frozenset( - _submodel_rate_keys("search_context_cost_per_query", entry.search_context_cost_per_query) - ) - | frozenset(_submodel_rate_keys("provider_specific_entry", entry.provider_specific_entry)) - ) - - -def _entry_has_rate_key(entry: CostMapEntry, rate_key: str) -> bool: - outer, _, inner = rate_key.partition(".") - if outer == "search_context_cost_per_query": - return f"{outer}.{inner}" in _submodel_rate_keys(outer, entry.search_context_cost_per_query) - if outer == "provider_specific_entry": - return f"{outer}.{inner}" in _submodel_rate_keys(outer, entry.provider_specific_entry) - value: Final[object] = getattr(entry, outer, None) - return value is not None - - -SERVICE_TIER_REQUEST_SHAPES: Final = frozenset( - {"openai_chat", "openai_responses", "bedrock_converse"} -) - - -COST_MAP_ADAPTER: Final = TypeAdapter(dict[str, CostMapEntry]) -COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType( - COST_MAP_ADAPTER.validate_python(json.loads(COST_MAP_PATH.read_text())) -) - -TIER_THRESHOLD_TOKENS: Final = 200_000 - - -class DeploymentSpec(BaseModel): - """A deployment-level fact from cases.json: when a map key needs a - registered deployment name that is not its provider model (or a - model_info.base_model pin), the matrix uses these instead of the defaults.""" - - model_config = ConfigDict(frozen=True) - - map_key: str - litellm_model: str | None = None - base_model: str | None = None - - -class ExpectedCell(BaseModel): - model_config = ConfigDict(frozen=True) - - spend: float - input_cost: float - output_cost: float - prompt_tokens: int - completion_tokens: int - - -class Case(BaseModel): - """One request/response shape from cases.json. - - ``family`` splits the matrix: ``pricing`` cases own cost keys (``owns``, - dotted subfield names allowed) or declare which keys they deliberately - leave absent (``fallback_for``) so every cost key in the map has exactly - one owning case; ``transport`` cases exercise counting/transport only and - run wherever they list membership. An exact-spend case names its models - implicitly by carrying one ``expected`` golden per map key; a recount - case (``exact_spend=False``) names them in ``models`` instead. The - feature flags drive request realism in ``_chat_body``.""" - - model_config = ConfigDict(frozen=True) - - name: str - family: Literal["pricing", "transport"] - usage: ScriptedUsage - usage_by_model: Mapping[str, ScriptedUsage] = Field(default_factory=lambda: MappingProxyType({})) - stream: bool = False - stream_usage: Literal["final_chunk", "absent"] = "final_chunk" - service_tier: Literal["flex", "priority"] | None = None - speed: Literal["fast"] | None = None - inference_geo: Literal["us"] | None = None - response_model_override: bool = False - exact_spend: bool = True - tool_call: bool = False - image_input: bool = False - audio_input: bool = False - audio_output: bool = False - video_input: bool = False - reasoning: bool = False - web_search: Literal["low", "medium", "high"] | None = None - google_maps: bool = False - file_search: bool = False - terminal: Literal["completed", "incomplete", "unvalidated", "prompt_blocked"] = "completed" - owns: tuple[str, ...] = () - fallback_for: tuple[str, ...] = () - expected: Mapping[str, ExpectedCell] = Field(default_factory=lambda: MappingProxyType({})) - models: tuple[str, ...] = () - - def applies_to(self, model: FrontierModel) -> bool: - if self.exact_spend: - return model.map_key in self.expected - return model.map_key in self.models - - def expected_for(self, model: FrontierModel) -> ExpectedCell: - return self.expected[model.map_key] - - def usage_for(self, map_key: str) -> ScriptedUsage: - return self.usage_by_model.get(map_key, self.usage) - - def scenario(self, scenario_id: str, model: FrontierModel, text: str) -> Scenario: - return Scenario( - scenario_id=scenario_id, - shape=model.shape, - usage=self.usage_for(model.map_key), - model=model.provider_model, - output=ScriptedOutput( - text=text, - response_model=model.override_model if self.response_model_override else None, - tool_call=ScriptedToolCall(name="get_weather", arguments=TOOL_CALL_ARGUMENTS) - if self.tool_call - else None, - terminal=self.terminal, - ), - stream_usage=self.stream_usage, - service_tier=self.service_tier, - speed=self.speed, - inference_geo=self.inference_geo, - ) - - -class _ProviderWiringRow(BaseModel): - model_config = ConfigDict(frozen=True) - - litellm_provider: str - mode: str - model_prefix: str | None - litellm_params: Mapping[str, str] - - -class _CasesFile(BaseModel): - model_config = ConfigDict(frozen=True) - - providers: tuple[_ProviderWiringRow, ...] = () - deployments: tuple[DeploymentSpec, ...] = () - cases: tuple[Case, ...] = () - - -CASES_FILE: Final = _CasesFile.model_validate(json.loads(CASES_PATH.read_text())) -CASES: Final[tuple[Case, ...]] = CASES_FILE.cases -_DEPLOYMENTS: Final[Mapping[str, DeploymentSpec]] = MappingProxyType( - {spec.map_key: spec for spec in CASES_FILE.deployments} -) - - -@dataclass(frozen=True, slots=True) -class _DeploymentDefaults: - """How a (litellm_provider, mode) pair maps to deployment defaults.""" - - model_prefix: str | None - litellm_params: Mapping[str, str] - - -def _deployment_defaults( - rows: tuple[_ProviderWiringRow, ...], -) -> Mapping[tuple[str, str], _DeploymentDefaults]: - return MappingProxyType( - { - (row.litellm_provider, row.mode): _DeploymentDefaults( - row.model_prefix, - MappingProxyType(dict(row.litellm_params)), - ) - for row in rows - } - ) - - -_DEPLOYMENT_DEFAULTS: Final[Mapping[tuple[str, str], _DeploymentDefaults]] = _deployment_defaults( - CASES_FILE.providers -) - - -@dataclass(frozen=True, slots=True) -class FrontierModel: - """One deployment under test, derived from a cost-map entry: the model_name - the suite registers, the provider-prefixed litellm model string, the - response shape the scripted upstream speaks, and the sibling map model the - response_model override case reports.""" - - model_name: str - litellm_model: str - shape: Shape - llm_provider: str - map_key: str - override_model: str | None = None - override_map_key: str | None = None - # Registered as model_info.base_model; when set, the provider-reported - # model loses to it and every case bills at this deployment's own rates. - base_model: str | None = None - litellm_params: Mapping[str, str] = MappingProxyType({}) - - @property - def rates(self) -> CostMapEntry: - return COST_MAP[self.map_key] - - @property - def override_rates(self) -> CostMapEntry: - # bedrock_converse responses carry no model field, so a reported-model - # override can never repoint pricing there, same as a base_model pin. - if ( - self.base_model is not None - or self.shape == "bedrock_converse" - or self.override_map_key is None - ): - return self.rates - return COST_MAP[self.override_map_key] - - @property - def provider_model(self) -> str: - """The bare provider-facing model name: litellm_model minus the provider - prefix and any routing segment (converse/, responses/).""" - return _provider_model(self.litellm_model) - - @property - def provider(self) -> str: - return self.rates.litellm_provider - - @property - def api_key(self) -> str: - # The scripted upstream ignores auth; a fixed bogus key proves the suite - # spends zero real provider calls. - return "sk-scripted-provider" - - -def _provider_model(litellm_model: str) -> str: - tail: Final = litellm_model.split("/")[1:] - return "/".join(tail[1:] if tail and tail[0] in ("converse", "responses") else tail) - - -def _litellm_model_for(map_key: str, defaults: _DeploymentDefaults) -> str: - if defaults.model_prefix is None: - return map_key - if map_key.startswith(f"{defaults.model_prefix}/"): - return map_key - return f"{defaults.model_prefix}/{map_key}" - - -def _resolve(litellm_model: str, mode: str) -> tuple[str, Shape]: - model, provider, _, _ = get_llm_provider(model=litellm_model) - llm_provider: Final = LlmProviders(provider) - if mode == "responses": - responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( - model=model, - provider=llm_provider, - ) - if isinstance(responses_config, OpenAIResponsesAPIConfig): - return provider, "openai_responses" - raise ValueError(f"no scripted renderer for {type(responses_config).__name__} ({litellm_model})") - config: Final = ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) - if isinstance(config, AmazonConverseConfig): - return provider, "bedrock_converse" - if isinstance(config, VertexGeminiConfig): - return provider, "gemini_generate" - if isinstance(config, AnthropicConfig): - return provider, "anthropic_messages" - if isinstance(config, (AzureOpenAIConfig, OpenAIGPTConfig)): - return provider, "openai_chat" - raise ValueError(f"no scripted renderer for {type(config).__name__} ({litellm_model})") - - -def _frontier() -> tuple[FrontierModel, ...]: - groups: Final[Mapping[tuple[str, str], tuple[str, ...]]] = MappingProxyType( - { - pair: tuple(sorted(k for k, e in COST_MAP.items() if (e.litellm_provider, e.mode) == pair)) - for pair in {(e.litellm_provider, e.mode) for e in COST_MAP.values()} - } - ) - models: list[FrontierModel] = [] # mutable-ok: accumulated once at import into a tuple - for map_key in sorted(COST_MAP): - entry = COST_MAP[map_key] - pair = (entry.litellm_provider, entry.mode) - defaults = _DEPLOYMENT_DEFAULTS.get(pair) - if defaults is None: - continue - siblings = groups[pair] - override_key = ( - siblings[(siblings.index(map_key) + 1) % len(siblings)] if len(siblings) > 1 else None - ) - override_litellm = ( - _litellm_model_for(override_key, defaults) if override_key is not None else None - ) - deployment = _DEPLOYMENTS.get(map_key) - litellm_model = ( - deployment.litellm_model - if deployment is not None and deployment.litellm_model is not None - else _litellm_model_for(map_key, defaults) - ) - llm_provider, shape = _resolve(litellm_model, entry.mode) - models.append( - FrontierModel( - model_name=f"cc-{map_key.replace('/', '-').replace(':', '-').replace('.', '-').lower()}", - litellm_model=litellm_model, - shape=shape, - llm_provider=llm_provider, - map_key=map_key, - override_model=( - _provider_model(override_litellm) - if override_litellm is not None - else None - ), - override_map_key=override_key, - base_model=deployment.base_model if deployment is not None else None, - litellm_params=defaults.litellm_params, - ) - ) - return tuple(models) - - -FRONTIER_MODELS: Final[tuple[FrontierModel, ...]] = _frontier() - -TOOL_CALL_ARGUMENTS: Final = json.dumps({ - "city": "Berlin", - "days": 7, - "units": "metric", - "notes": "filler " * 30, -}) - - -def cases_for(model: FrontierModel) -> tuple[Case, ...]: - return tuple(case for case in CASES if case.applies_to(model)) - - -def recount_cost( - model: FrontierModel, case: Case, prompt_tokens: int, completion_tokens: int -) -> float: - """What the proxy's own token recount should cost at the case's rates, - without pinning the tokenizer's exact counts.""" - rates: Final = model.override_rates if case.response_model_override else model.rates - return prompt_tokens * (rates.input_cost_per_token or 0.0) + completion_tokens * ( - rates.output_cost_per_token or 0.0 - ) - - -def _png_chunk(tag: bytes, payload: bytes) -> bytes: - return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", zlib.crc32(tag + payload)) - - -def audio_input_data_url() -> str: - """A deterministic 0.5 s 16-bit PCM WAV (8 kHz, 220 Hz sine) as a data - URL, small enough to stay a fixture but real audio to the provider.""" - frames: Final = b"".join( - struct.pack(" str: - """A deterministic mp4-looking blob (ftyp box plus a fixed mdat payload) - as a data URL; only the media type and bytes matter to the response.""" - ftyp: Final = struct.pack(">I4s4sI4s4s", 24, b"ftyp", b"isom", 0x200, b"isom", b"iso6") - mdat_payload: Final = bytes((i * 7 + 13) % 256 for i in range(4096)) - mdat: Final = struct.pack(">I4s", 8 + len(mdat_payload), b"mdat") + mdat_payload - return "data:video/mp4;base64," + base64.b64encode(ftyp + mdat).decode() - - -def image_input_data_url() -> str: - """A deterministic 256x256 RGB noise PNG as a data URL; noise compresses - poorly on purpose so the base64 payload stays well above 100 KB and would - blow up the prompt recount if the URL were ever tokenized as text.""" - rng: Final = random.Random(0) - side: Final = 256 - raw: Final = b"".join( - b"\x00" + rng.randbytes(side * 3) for _ in range(side) - ) - png: Final = ( - b"\x89PNG\r\n\x1a\n" - + _png_chunk(b"IHDR", struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)) - + _png_chunk(b"IDAT", zlib.compress(raw)) - + _png_chunk(b"IEND", b"") - ) - return "data:image/png;base64," + base64.b64encode(png).decode() - - -IMAGE_INPUT_DATA_URL: Final = image_input_data_url() -AUDIO_INPUT_DATA_URL: Final = audio_input_data_url() -VIDEO_INPUT_DATA_URL: Final = video_input_data_url() - - -def matrix_data_errors() -> tuple[str, ...]: - """Consistency findings for the data files, as human-readable strings. - - Called at collection time by the integration suite, so a map key named by a case - but absent from cost_map.json fails the suite's collection loudly. - """ - unknown_deployments: Final = sorted( - spec.map_key for spec in CASES_FILE.deployments if spec.map_key not in COST_MAP - ) - unknown_case_models: Final = sorted( - { - map_key - for case in CASES - for map_key in (*case.expected, *case.models) - if map_key not in COST_MAP - } - ) - misshapen_cases: Final = sorted( - case.name - for case in CASES - if case.exact_spend == bool(case.models) or case.exact_spend != bool(case.expected) - ) - all_pairs: Final = frozenset( - (map_key, key) - for map_key, entry in COST_MAP.items() - for key in _entry_rate_keys(entry) - ) - owned_pairs: Final = tuple( - (map_key, key) - for case in CASES - if case.family == "pricing" - for map_key in case.expected - for key in case.owns - if map_key in COST_MAP and _entry_has_rate_key(COST_MAP[map_key], key) - ) - unowned_pairs: Final = sorted( - f"{map_key}:{key}" for map_key, key in all_pairs - frozenset(owned_pairs) - ) - duplicate_pairs: Final = sorted( - f"{map_key}:{key}" - for map_key, key in set(owned_pairs) - if owned_pairs.count((map_key, key)) > 1 - ) - owns_without_holder: Final = sorted( - f"{case.name}:{key}" - for case in CASES - for key in case.owns - if not any( - map_key in COST_MAP and _entry_has_rate_key(COST_MAP[map_key], key) - for map_key in case.expected - ) - ) - fallback_violations: Final = sorted( - f"{case.name}:{map_key}:{key}" - for case in CASES - for key in case.fallback_for - for map_key in (*case.expected, *case.models) - if map_key in COST_MAP and _entry_has_rate_key(COST_MAP[map_key], key) - ) - family_violations: Final = sorted( - case.name - for case in CASES - if (case.family == "transport") != (not case.owns and not case.fallback_for) - ) - missing_provider_rows: Final = sorted( - f"cost_map entry {map_key} has no providers row for " - f"(litellm_provider={entry.litellm_provider}, mode={entry.mode}); " - f"add a providers row in cases.json" - for map_key, entry in COST_MAP.items() - if (entry.litellm_provider, entry.mode) not in _DEPLOYMENT_DEFAULTS - ) - input_rates: Final = tuple(entry.input_cost_per_token for entry in COST_MAP.values()) - findings: Final = ( - ( - f"deployments entries name map keys absent from cost_map.json: {unknown_deployments}" - if unknown_deployments - else None - ), - ( - f"case expected/models name map keys absent from cost_map.json: {unknown_case_models}" - if unknown_case_models - else None - ), - ( - f"cases must carry expected xor models (exact_spend matches the field): {misshapen_cases}" - if misshapen_cases - else None - ), - ( - "two cost_map entries share input_cost_per_token; the suite relies on " - "distinct rates so a wrong-model bill can never coincidentally match" - if len(input_rates) != len(set(input_rates)) - else None - ), - ( - f"(model, rate key) pairs with no owning case: {unowned_pairs}" - if unowned_pairs - else None - ), - ( - f"(model, rate key) pairs owned by more than one case: {duplicate_pairs}" - if duplicate_pairs - else None - ), - ( - f"owns keys absent on all of the case's expected models: {owns_without_holder}" - if owns_without_holder - else None - ), - ( - f"fallback_for keys a case's models actually carry: {fallback_violations}" - if fallback_violations - else None - ), - ( - f"cases with owns/fallback_for inconsistent with family: {family_violations}" - if family_violations - else None - ), - ( - f"cost_map entries without providers rows: {missing_provider_rows}" - if missing_provider_rows - else None - ), - ) - return tuple(finding for finding in findings if finding is not None) diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py new file mode 100644 index 00000000000..6af95f995ff --- /dev/null +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +from collections.abc import Mapping +from pathlib import Path +from types import MappingProxyType +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, JsonValue + +CASES_PATH: Final = Path(__file__).resolve().parent / "cost_tracking_cases.json" + + +class SearchContextCostPerQuery(BaseModel): + model_config = ConfigDict(frozen=True) + + search_context_size_low: float | None = None + search_context_size_medium: float | None = None + search_context_size_high: float | None = None + + +class ProviderSpecificEntry(BaseModel): + model_config = ConfigDict(frozen=True) + + fast: float | None = None + us: float | None = None + + +class CostMapEntry(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + litellm_provider: str + mode: str + max_tokens: int | None = None + max_input_tokens: int | None = None + max_output_tokens: int | None = None + supports_function_calling: bool | None = None + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + cache_creation_input_token_cost_above_1hr: float | None = None + cache_read_input_token_cost_above_200k_tokens: float | None = None + cache_creation_input_token_cost_above_200k_tokens: float | None = None + output_cost_per_reasoning_token: float | None = None + input_cost_per_audio_token: float | None = None + output_cost_per_audio_token: float | None = None + input_cost_per_image_token: float | None = None + input_cost_per_video_token: float | None = None + input_cost_per_token_above_200k_tokens: float | None = None + output_cost_per_token_above_200k_tokens: float | None = None + input_cost_per_token_flex: float | None = None + output_cost_per_token_flex: float | None = None + input_cost_per_token_priority: float | None = None + output_cost_per_token_priority: float | None = None + search_context_cost_per_query: SearchContextCostPerQuery | None = None + web_search_billing_unit: str | None = None + google_maps_grounding_cost_per_query: float | None = None + file_search_cost_per_1k_calls: float | None = None + provider_specific_entry: ProviderSpecificEntry | None = None + + +class Deployment(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + model: str | None = None + base_model: str | None = None + + +class JsonResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + content_type: Literal["application/json"] + body: dict[str, JsonValue] + + +class SseResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + content_type: Literal["text/event-stream"] + frames: tuple[str, ...] + + +class EventStreamEvent(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + event_type: str + payload: dict[str, JsonValue] + + +class EventStreamResponse(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + content_type: Literal["application/vnd.amazon.eventstream"] + events: tuple[EventStreamEvent, ...] + + +StoredResponse: TypeAlias = Annotated[ + JsonResponse | SseResponse | EventStreamResponse, + Field(discriminator="content_type"), +] + + +class ExactExpected(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + spend: float + input_cost: float + output_cost: float + prompt_tokens: int + completion_tokens: int + + +class RecountRates(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + input_cost_per_token: float + output_cost_per_token: float + + +class RecountExpected(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + recount: RecountRates + + +Expected: TypeAlias = ExactExpected | RecountExpected + + +class CostTrackingTestCase(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + covers: str + model: str + deployment: Deployment | None = None + request: dict[str, JsonValue] + response: StoredResponse + expected: Expected + + @property + def rates(self) -> CostMapEntry: + return COST_MAP[self.model] + + @property + def litellm_model(self) -> str: + provider: Final = self.rates.litellm_provider + prefix: Final = ( + "openai" + if provider == "openai" and self.rates.mode == "chat" + else "openai/responses" + if provider == "openai" + else _PROVIDER_PREFIXES.get(provider) + ) + if prefix is None: + raise ValueError(f"unsupported cost-map provider {provider} for {self.model}") + return self.deployment.model if self.deployment and self.deployment.model is not None else ( + self.model if prefix == "" else f"{prefix}/{self.model}" + ) + + @property + def litellm_params(self) -> Mapping[str, str]: + return _LITELLM_PARAMS[self.rates.litellm_provider] + + @property + def api_key(self) -> str: + return "sk-scripted-provider" + + @property + def base_model(self) -> str | None: + return self.deployment.base_model if self.deployment else None + + +class _CasesFile(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + cost_map: dict[str, CostMapEntry] + cases: tuple[CostTrackingTestCase, ...] + + +_PROVIDER_PREFIXES: Final[Mapping[str, str]] = MappingProxyType( + { + "anthropic": "anthropic", + "bedrock_converse": "bedrock/converse", + "vertex_ai-language-models": "vertex_ai", + "gemini": "", + "together_ai": "", + "fireworks_ai": "", + "azure": "", + } +) +_LITELLM_PARAMS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( + { + "anthropic": MappingProxyType({}), + "bedrock_converse": MappingProxyType( + { + "aws_access_key_id": "AKIASCRIPTEDPROVIDER", + "aws_secret_access_key": "scripted-secret", + "aws_region_name": "us-east-1", + } + ), + "vertex_ai-language-models": MappingProxyType( + {"vertex_project": "cc-scripted-project", "vertex_location": "us-central1"} + ), + "gemini": MappingProxyType({}), + "together_ai": MappingProxyType({}), + "fireworks_ai": MappingProxyType({}), + "azure": MappingProxyType({"api_version": "2025-04-01-preview"}), + "openai": MappingProxyType({}), + } +) + +_LOADED: Final = _CasesFile.model_validate_json(CASES_PATH.read_bytes()) +COST_MAP: Final[Mapping[str, CostMapEntry]] = MappingProxyType(dict(_LOADED.cost_map)) +CASES: Final[tuple[CostTrackingTestCase, ...]] = _LOADED.cases +_LITELLM_MODELS: Final = tuple(case.litellm_model for case in CASES) + + +def data_errors() -> tuple[str, ...]: + case_models: Final = frozenset(case.model for case in CASES) + unknown_models: Final = sorted(case.model for case in CASES if case.model not in COST_MAP) + missing_cases: Final = sorted(model for model in COST_MAP if model not in case_models) + duplicate_names: Final = sorted( + name for name in {case.name for case in CASES} if sum(case.name == name for case in CASES) > 1 + ) + input_rates: Final = tuple( + (entry.input_cost_per_token, model) for model, entry in COST_MAP.items() + ) + shared_input_rates: Final = sorted( + f"{rate}: {tuple(model for value, model in input_rates if value == rate)}" + for rate in {value for value, _ in input_rates if value is not None} + if sum(value == rate for value, _ in input_rates) > 1 + ) + recount_mismatches: Final = sorted( + case.name + for case in CASES + if isinstance(case.expected, RecountExpected) + and case.model in COST_MAP + and ( + case.expected.recount.input_cost_per_token != (COST_MAP[case.model].input_cost_per_token or 0.0) + or case.expected.recount.output_cost_per_token != (COST_MAP[case.model].output_cost_per_token or 0.0) + ) + ) + return tuple( + message + for message in ( + f"case models absent from cost_map: {unknown_models}" if unknown_models else None, + f"cost-map entries without cases: {missing_cases}" if missing_cases else None, + f"duplicate case names: {duplicate_names}" if duplicate_names else None, + f"cost-map entries share input_cost_per_token: {shared_input_rates}" if shared_input_rates else None, + f"recount rates differ from cost-map rates: {recount_mismatches}" if recount_mismatches else None, + ) + if message is not None + ) diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json new file mode 100644 index 00000000000..3627774816f --- /dev/null +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -0,0 +1,25658 @@ +{ + "cost_map": { + "gpt-5.6": { + "cache_read_input_token_cost": 1.75e-07, + "input_cost_per_audio_token": 4e-05, + "input_cost_per_token": 1.75e-06, + "input_cost_per_token_flex": 8.75e-07, + "input_cost_per_token_priority": 3.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 8e-05, + "output_cost_per_reasoning_token": 1.6e-05, + "output_cost_per_token": 1.4e-05, + "output_cost_per_token_flex": 7e-06, + "output_cost_per_token_priority": 2.8e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "gpt-5.4-mini": { + "cache_read_input_token_cost": 3.5e-08, + "input_cost_per_audio_token": 1e-05, + "input_cost_per_token": 3.5e-07, + "input_cost_per_token_flex": 1.75e-07, + "input_cost_per_token_priority": 7e-07, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_reasoning_token": 3.2e-06, + "output_cost_per_token": 2.8e-06, + "output_cost_per_token_flex": 1.4e-06, + "output_cost_per_token_priority": 5.6e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "azure/gpt-5.6": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_audio_token": 4.1e-05, + "input_cost_per_token": 1.8e-06, + "input_cost_per_token_flex": 9e-07, + "input_cost_per_token_priority": 3.6e-06, + "litellm_provider": "azure", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 8.2e-05, + "output_cost_per_reasoning_token": 1.65e-05, + "output_cost_per_token": 1.44e-05, + "output_cost_per_token_flex": 7.2e-06, + "output_cost_per_token_priority": 2.88e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "azure/gpt-5.4-mini": { + "cache_read_input_token_cost": 3.6e-08, + "input_cost_per_audio_token": 1.05e-05, + "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, + "input_cost_per_token_priority": 7.2e-07, + "litellm_provider": "azure", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 2.1e-05, + "output_cost_per_reasoning_token": 3.3e-06, + "output_cost_per_token": 2.88e-06, + "output_cost_per_token_flex": 1.44e-06, + "output_cost_per_token_priority": 5.76e-06, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "gpt-5.3-codex": { + "cache_read_input_token_cost": 1.5e-07, + "file_search_cost_per_1k_calls": 0.0025, + "input_cost_per_token": 1.5e-06, + "input_cost_per_token_flex": 7.5e-07, + "input_cost_per_token_priority": 3e-06, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "responses", + "output_cost_per_reasoning_token": 1.3e-05, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_flex": 6e-06, + "output_cost_per_token_priority": 2.4e-05, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "gpt-5.5-pro": { + "cache_read_input_token_cost": 1.5e-06, + "file_search_cost_per_1k_calls": 0.0025, + "input_cost_per_token": 1.5e-05, + "input_cost_per_token_flex": 7.5e-06, + "input_cost_per_token_priority": 3e-05, + "litellm_provider": "openai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "responses", + "output_cost_per_reasoning_token": 0.00013, + "output_cost_per_token": 0.00012, + "output_cost_per_token_flex": 6e-05, + "output_cost_per_token_priority": 0.00024, + "search_context_cost_per_query": { + "search_context_size_low": 0.01, + "search_context_size_medium": 0.0125, + "search_context_size_high": 0.015 + }, + "supports_function_calling": true + }, + "claude-opus-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_creation_input_token_cost_above_200k_tokens": 1.25e-05, + "cache_read_input_token_cost": 5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_token_above_200k_tokens": 1e-05, + "input_cost_per_token_priority": 6.25e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "output_cost_per_token_above_200k_tokens": 3.75e-05, + "output_cost_per_token_priority": 3.125e-05, + "provider_specific_entry": { + "fast": 6.0, + "us": 1.1 + }, + "search_context_cost_per_query": { + "search_context_size_medium": 0.01 + }, + "supports_function_calling": true + }, + "claude-sonnet-5": { + "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost": 3e-07, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_above_200k_tokens": 6e-06, + "input_cost_per_token_priority": 3.75e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.5e-05, + "output_cost_per_token_above_200k_tokens": 2.25e-05, + "output_cost_per_token_priority": 1.875e-05, + "provider_specific_entry": { + "us": 1.1 + }, + "search_context_cost_per_query": { + "search_context_size_medium": 0.01 + }, + "supports_function_calling": true + }, + "claude-haiku-4-5": { + "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "input_cost_per_token_priority": 1.25e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 5e-06, + "output_cost_per_token_priority": 6.25e-06, + "provider_specific_entry": { + "us": 1.1 + }, + "search_context_cost_per_query": { + "search_context_size_medium": 0.01 + }, + "supports_function_calling": true + }, + "us.anthropic.claude-opus-5-v1:0": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, + "cache_creation_input_token_cost_above_200k_tokens": 1.375e-05, + "cache_read_input_token_cost": 5.5e-07, + "cache_read_input_token_cost_above_200k_tokens": 1.1e-06, + "input_cost_per_token": 5.5e-06, + "input_cost_per_token_above_200k_tokens": 1.1e-05, + "input_cost_per_token_flex": 2.75e-06, + "input_cost_per_token_priority": 6.875e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "output_cost_per_token_above_200k_tokens": 4.125e-05, + "output_cost_per_token_flex": 1.375e-05, + "output_cost_per_token_priority": 3.4375e-05, + "supports_function_calling": true + }, + "anthropic.claude-sonnet-5-v1:0": { + "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, + "cache_read_input_token_cost": 3.3e-07, + "input_cost_per_token": 3.3e-06, + "input_cost_per_token_flex": 1.65e-06, + "input_cost_per_token_priority": 4.125e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1.65e-05, + "output_cost_per_token_flex": 8.25e-06, + "output_cost_per_token_priority": 2.0625e-05, + "supports_function_calling": true + }, + "meta.llama4-maverick-17b-instruct-v1:0": { + "input_cost_per_token": 2.4e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 9.7e-07, + "supports_function_calling": true + }, + "gemini/gemini-3.1-pro": { + "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_above_200k_tokens": 4e-07, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 2.6e-06, + "input_cost_per_image_token": 2.2e-06, + "input_cost_per_token": 2e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + "input_cost_per_token_flex": 1e-06, + "input_cost_per_token_priority": 2.5e-06, + "input_cost_per_video_token": 2.4e-06, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 1.3e-05, + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_above_200k_tokens": 1.8e-05, + "output_cost_per_token_flex": 6e-06, + "output_cost_per_token_priority": 1.5e-05, + "search_context_cost_per_query": { + "search_context_size_medium": 0.035 + }, + "supports_function_calling": true, + "web_search_billing_unit": "per_query" + }, + "gemini/gemini-3.8-flash": { + "cache_read_input_token_cost": 5e-08, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 1e-06, + "input_cost_per_image_token": 5.5e-07, + "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, + "input_cost_per_token_priority": 6.25e-07, + "input_cost_per_video_token": 6e-07, + "litellm_provider": "gemini", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 6e-06, + "output_cost_per_reasoning_token": 3.5e-06, + "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, + "output_cost_per_token_priority": 3.75e-06, + "search_context_cost_per_query": { + "search_context_size_medium": 0.035 + }, + "supports_function_calling": true, + "web_search_billing_unit": "per_prompt" + }, + "gemini-3.1-pro": { + "cache_read_input_token_cost": 2.1e-07, + "cache_read_input_token_cost_above_200k_tokens": 4.2e-07, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 2.7e-06, + "input_cost_per_image_token": 2.3e-06, + "input_cost_per_token": 2.1e-06, + "input_cost_per_token_above_200k_tokens": 4.2e-06, + "input_cost_per_token_flex": 1.05e-06, + "input_cost_per_token_priority": 2.625e-06, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_reasoning_token": 1.35e-05, + "output_cost_per_token": 1.26e-05, + "output_cost_per_token_above_200k_tokens": 1.89e-05, + "output_cost_per_token_flex": 6.3e-06, + "output_cost_per_token_priority": 1.575e-05, + "search_context_cost_per_query": { + "search_context_size_medium": 0.035 + }, + "supports_function_calling": true, + "web_search_billing_unit": "per_query" + }, + "gemini-3.8-flash": { + "cache_read_input_token_cost": 5.2e-08, + "google_maps_grounding_cost_per_query": 0.025, + "input_cost_per_audio_token": 1.04e-06, + "input_cost_per_token": 5.2e-07, + "input_cost_per_token_flex": 2.6e-07, + "input_cost_per_token_priority": 6.5e-07, + "input_cost_per_video_token": 6.2e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_audio_token": 6.24e-06, + "output_cost_per_token": 3.12e-06, + "output_cost_per_token_flex": 1.56e-06, + "output_cost_per_token_priority": 3.9e-06, + "search_context_cost_per_query": { + "search_context_size_medium": 0.035 + }, + "supports_function_calling": true, + "web_search_billing_unit": "per_prompt" + }, + "together_ai/moonshotai/Kimi-K3": { + "input_cost_per_token": 1.15e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.45e-06, + "supports_function_calling": true + }, + "together_ai/zai-org/GLM-5.3": { + "input_cost_per_token": 5.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.2e-06, + "supports_function_calling": true + }, + "fireworks_ai/accounts/fireworks/models/kimi-k3": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 6e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true + }, + "fireworks_ai/accounts/fireworks/models/qwen3p8-max": { + "cache_read_input_token_cost": 9e-08, + "input_cost_per_token": 9e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 400000, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3.6e-06, + "supports_function_calling": true + } + }, + "cases": [ + { + "name": "anthropic.claude-sonnet-5-v1:0-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9aad4de0556c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 9aad4de0556c" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "708bfb28f35a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 708bfb28f35a" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 640, + "outputTokens": 380, + "totalTokens": 13308, + "cacheReadInputTokens": 12288 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.01243704, + "input_cost": 0.00616704, + "output_cost": 0.00627, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-cache_write_5m", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "08c49d1b837c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 08c49d1b837c" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 512, + "outputTokens": 350, + "totalTokens": 10078, + "cacheWriteInputTokens": 9216, + "cacheDetails": [ + { + "inputTokens": 9216, + "ttl": "5m" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.0454806, + "input_cost": 0.0397056, + "output_cost": 0.005775, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-cache_write_1h", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b96166d8affb summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer b96166d8affb" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 512, + "outputTokens": 350, + "totalTokens": 10078, + "cacheWriteInputTokens": 9216, + "cacheDetails": [ + { + "inputTokens": 2048, + "ttl": "5m" + }, + { + "inputTokens": 7168, + "ttl": "1h" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.0632214, + "input_cost": 0.0574464, + "output_cost": 0.005775, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "41dbeb5496b2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 41dbeb5496b2" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + }, + "serviceTier": { + "type": "flex" + } + } + }, + "expected": { + "spend": 0.006435, + "input_cost": 0.003036, + "output_cost": 0.003399, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f6c242da055f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer f6c242da055f" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + }, + "serviceTier": { + "type": "priority" + } + } + }, + "expected": { + "spend": 0.0160875, + "input_cost": 0.00759, + "output_cost": 0.0084975, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a9257967d38a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer a9257967d38a" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ca62b8bbf5b6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer ca62b8bbf5b6" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05 + } + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6f00980cd47c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockStart", + "payload": { + "start": { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler fil" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ler filler filler filler filler filler filler filler filler filler filler filler filler fi" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ller filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "tool_use" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05 + } + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2a972e053197 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 2a972e053197" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.3e-06, + "output_cost_per_token": 1.65e-05 + } + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c300c8153393 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer c300c8153393" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4c599e93dfba summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 4c599e93dfba" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c40237f9541a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather", + "input": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1727b8128120 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockStart", + "payload": { + "start": { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler fil" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ler filler filler filler filler filler filler filler filler filler filler filler filler fi" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ller filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "tool_use" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "anthropic.claude-sonnet-5-v1:0-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "anthropic.claude-sonnet-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "649a7735f7cb summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 649a7735f7cb" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 11468, + "cacheReadInputTokens": 6144, + "cacheWriteInputTokens": 3072, + "cacheDetails": [ + { + "inputTokens": 2048, + "ttl": "5m" + }, + { + "inputTokens": 1024, + "ttl": "1h" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.03010392, + "input_cost": 0.02330592, + "output_cost": 0.006798, + "prompt_tokens": 11056, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "28b0c4ce80d6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788217, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 28b0c4ce80d6" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "33fdcb306184 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788218, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 33fdcb306184" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.001767168, + "input_cost": 0.000672768, + "output_cost": 0.0010944, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "azure-gpt-5.4-mini-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "28a136ca9579 summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788220, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 28a136ca9579" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1546, + "completion_tokens": 210, + "total_tokens": 1756, + "prompt_tokens_details": { + "audio_tokens": 1450 + } + } + } + }, + "expected": { + "spend": 0.01586436, + "input_cost": 0.01525956, + "output_cost": 0.0006048, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "azure-gpt-5.4-mini-audio_output", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2bebbaa4e254 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "modalities": [ + "text", + "audio" + ], + "audio": { + "voice": "alloy", + "format": "pcm16" + }, + "allowed_openai_params": [ + "modalities", + "audio" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788222, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 2bebbaa4e254" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 220, + "completion_tokens": 1300, + "total_tokens": 1520, + "completion_tokens_details": { + "audio_tokens": 1120 + } + } + } + }, + "expected": { + "spend": 0.0241176, + "input_cost": 7.92e-05, + "output_cost": 0.0240384, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + }, + { + "name": "azure-gpt-5.4-mini-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6af41b14ef04 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788222, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 6af41b14ef04" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1240, + "completion_tokens": 4040, + "total_tokens": 5280, + "completion_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.0135432, + "input_cost": 0.0004464, + "output_cost": 0.0130968, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "azure-gpt-5.4-mini-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "60a03b8b6237 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788225, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 60a03b8b6237" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "flex" + } + }, + "expected": { + "spend": 0.00092448, + "input_cost": 0.0003312, + "output_cost": 0.00059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3922bd062f4a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788226, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 3922bd062f4a" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "priority" + } + }, + "expected": { + "spend": 0.00369792, + "input_cost": 0.0013248, + "output_cost": 0.00237312, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "26430574f63b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "medium" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788211, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 26430574f63b", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.01434896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-web_search_low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5792eab53e4c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "low" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788213, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 5792eab53e4c", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.01184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-web_search_high", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7a7fc7488611 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "high" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788215, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 7a7fc7488611", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.01684896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "35a730eefc00 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 35a730eefc00\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"cc-pinned-deployment\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "969d5ff8918e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788216, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788216, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 969d5ff8918e\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788216, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 2.88e-06 + } + } + }, + { + "name": "azure-gpt-5.4-mini-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "db6294a8264b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 2.88e-06 + } + } + }, + { + "name": "azure-gpt-5.4-mini-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "524f8c567f64 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788220, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788220, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 524f8c567f64\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788220, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.6e-07, + "output_cost_per_token": 2.88e-06 + } + } + }, + { + "name": "azure-gpt-5.4-mini-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "109128998398 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788221, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 109128998398" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e5bffdbd65e2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer e5bffdbd65e2\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "75003151c7de summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788223, + "model": "cc-pinned-deployment", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e3e7c45697d3 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788225, \"model\": \"cc-pinned-deployment\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.4-mini-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.4-mini", + "deployment": { + "model": "azure/cc-pinned-deployment", + "base_model": "azure/gpt-5.4-mini" + }, + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "906fb0b08ba9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 906fb0b08ba9\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"cc-pinned-deployment\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"cc-pinned-deployment\", \"choices\": [], \"usage\": {\"prompt_tokens\": 8314, \"completion_tokens\": 1592, \"total_tokens\": 9906, \"prompt_tokens_details\": {\"cached_tokens\": 6144, \"audio_tokens\": 330}, \"completion_tokens_details\": {\"reasoning_tokens\": 900, \"audio_tokens\": 280}}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.014385144, + "input_cost": 0.004348584, + "output_cost": 0.01003656, + "prompt_tokens": 8314, + "completion_tokens": 1592 + } + }, + { + "name": "azure-gpt-5.6-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7dc90bf2ab07 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788212, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 7dc90bf2ab07" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c84b90d4fd99 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788214, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer c84b90d4fd99" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.00883584, + "input_cost": 0.00336384, + "output_cost": 0.005472, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "azure-gpt-5.6-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1744b6a5bab3 summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788215, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 1744b6a5bab3" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1546, + "completion_tokens": 210, + "total_tokens": 1756, + "prompt_tokens_details": { + "audio_tokens": 1450 + } + } + } + }, + "expected": { + "spend": 0.0626468, + "input_cost": 0.0596228, + "output_cost": 0.003024, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "azure-gpt-5.6-audio_output", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "dfb52830c629 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "modalities": [ + "text", + "audio" + ], + "audio": { + "voice": "alloy", + "format": "pcm16" + }, + "allowed_openai_params": [ + "modalities", + "audio" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788217, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer dfb52830c629" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 220, + "completion_tokens": 1300, + "total_tokens": 1520, + "completion_tokens_details": { + "audio_tokens": 1120 + } + } + } + }, + "expected": { + "spend": 0.094828, + "input_cost": 0.000396, + "output_cost": 0.094432, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + }, + { + "name": "azure-gpt-5.6-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5c6865969ae9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788218, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 5c6865969ae9" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1240, + "completion_tokens": 4040, + "total_tokens": 5280, + "completion_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.067716, + "input_cost": 0.002232, + "output_cost": 0.065484, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "azure-gpt-5.6-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b67dcd189cdd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788221, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer b67dcd189cdd" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "flex" + } + }, + "expected": { + "spend": 0.0046224, + "input_cost": 0.001656, + "output_cost": 0.0029664, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "01c77d1ef23d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788222, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 01c77d1ef23d" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "priority" + } + }, + "expected": { + "spend": 0.0184896, + "input_cost": 0.006624, + "output_cost": 0.0118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d4efeea706ac summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "medium" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788222, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer d4efeea706ac", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0217448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-web_search_low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "73458bfd2358 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "low" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788225, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 73458bfd2358", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0192448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-web_search_high", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0aebd59315f2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "high" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788226, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 0aebd59315f2", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0242448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9df3f46fd138 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 9df3f46fd138\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788211, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b54d5959e61f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788213, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788213, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer b54d5959e61f\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788213, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.8e-06, + "output_cost_per_token": 1.44e-05 + } + } + }, + { + "name": "azure-gpt-5.6-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "23eb3226fc23 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788215, \"model\": \"gpt-5.6\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_item.added\ndata: {\"type\": \"response.output_item.added\", \"output_index\": 0, \"item\": {\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"\", \"status\": \"in_progress\"}}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.function_call_arguments.done\ndata: {\"type\": \"response.function_call_arguments.done\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788215, \"status\": \"completed\", \"model\": \"gpt-5.6\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.8e-06, + "output_cost_per_token": 1.44e-05 + } + } + }, + { + "name": "azure-gpt-5.6-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "fb83549ab4c5 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer fb83549ab4c5\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788215, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.8e-06, + "output_cost_per_token": 1.44e-05 + } + } + }, + { + "name": "azure-gpt-5.6-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "74e949a94e0f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788216, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 74e949a94e0f" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "af89dddadd12 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer af89dddadd12\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788218, \"model\": \"gpt-5.4-mini\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.00184896, + "input_cost": 0.0006624, + "output_cost": 0.00118656, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "30d4deb9b74f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788220, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "function_call", + "id": "fc_$REQUEST_ID", + "call_id": "call_$REQUEST_ID", + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}", + "status": "completed" + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ef44a3525238 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788221, \"model\": \"gpt-5.6\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_item.added\ndata: {\"type\": \"response.output_item.added\", \"output_index\": 0, \"item\": {\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"\", \"status\": \"in_progress\"}}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.function_call_arguments.done\ndata: {\"type\": \"response.function_call_arguments.done\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788221, \"status\": \"completed\", \"model\": \"gpt-5.6\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.0092448, + "input_cost": 0.003312, + "output_cost": 0.0059328, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "azure-gpt-5.6-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "azure/gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e440709770ad summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer e440709770ad\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788221, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 8314, \"completion_tokens\": 1592, \"total_tokens\": 9906, \"prompt_tokens_details\": {\"cached_tokens\": 6144, \"audio_tokens\": 330}, \"completion_tokens_details\": {\"reasoning_tokens\": 900, \"audio_tokens\": 280}}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.06169072, + "input_cost": 0.01794792, + "output_cost": 0.0437428, + "prompt_tokens": 8314, + "completion_tokens": 1592 + } + }, + { + "name": "claude-haiku-4-5-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "83d8e1f3f711 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer 83d8e1f3f711" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e56cd6ddbc3b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer e56cd6ddbc3b" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 640, + "output_tokens": 380, + "cache_read_input_tokens": 12288 + } + } + }, + "expected": { + "spend": 0.0037688, + "input_cost": 0.0018688, + "output_cost": 0.0019, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "claude-haiku-4-5-cache_write_5m", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "aead4d429a63 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer aead4d429a63" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 9216, + "ephemeral_1h_input_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.013782, + "input_cost": 0.012032, + "output_cost": 0.00175, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "claude-haiku-4-5-cache_write_1h", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8defd838f26f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer 8defd838f26f" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 2048, + "ephemeral_1h_input_tokens": 7168 + } + } + } + }, + "expected": { + "spend": 0.019158, + "input_cost": 0.017408, + "output_cost": 0.00175, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "claude-haiku-4-5-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f7dcd0281161 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer f7dcd0281161" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "service_tier": "priority" + } + } + }, + "expected": { + "spend": 0.004875, + "input_cost": 0.0023, + "output_cost": 0.002575, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-anthropic_us_inference", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1c0a1a2e155f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer 1c0a1a2e155f" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "inference_geo": "us" + } + } + }, + "expected": { + "spend": 0.00429, + "input_cost": 0.002024, + "output_cost": 0.002266, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "540998778abd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer 540998778abd" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": { + "web_search_requests": 3 + } + } + } + }, + "expected": { + "spend": 0.0339, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8feb52d222c0 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 8feb52d222c0\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ac21e9843010 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer ac21e9843010\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06 + } + } + }, + { + "name": "claude-haiku-4-5-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "596ca026b176 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"tool_use\", \"id\": \"toolu_$REQUEST_ID\", \"name\": \"get_weather\", \"input\": {}}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"tool_use\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06 + } + } + }, + { + "name": "claude-haiku-4-5-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "89c83ea0f121 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 89c83ea0f121\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 5e-06 + } + } + }, + { + "name": "claude-haiku-4-5-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d5df85778fb1 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer d5df85778fb1" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2f3ca8c25a81 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 2f3ca8c25a81\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "74403961022c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "tool_use", + "id": "toolu_$REQUEST_ID", + "name": "get_weather", + "input": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + ], + "stop_reason": "tool_use", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5a30a53bb4d6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"tool_use\", \"id\": \"toolu_$REQUEST_ID\", \"name\": \"get_weather\", \"input\": {}}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"tool_use\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-haiku-4-5-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-haiku-4-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f89827fda6c5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840, \"cache_read_input_tokens\": 6144, \"cache_creation_input_tokens\": 3072, \"cache_creation\": {\"ephemeral_5m_input_tokens\": 2048, \"ephemeral_1h_input_tokens\": 1024}}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer f89827fda6c5\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0091224, + "input_cost": 0.0070624, + "output_cost": 0.00206, + "prompt_tokens": 11056, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d4916e93889c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer d4916e93889c" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b83799f51ed7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer b83799f51ed7" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 640, + "output_tokens": 380, + "cache_read_input_tokens": 12288 + } + } + }, + "expected": { + "spend": 0.018844, + "input_cost": 0.009344, + "output_cost": 0.0095, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "claude-opus-5-cache_write_5m", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5cfdc176130a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer 5cfdc176130a" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 9216, + "ephemeral_1h_input_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.06891, + "input_cost": 0.06016, + "output_cost": 0.00875, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "claude-opus-5-cache_write_1h", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6f9107c3b3ff summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer 6f9107c3b3ff" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 2048, + "ephemeral_1h_input_tokens": 7168 + } + } + } + }, + "expected": { + "spend": 0.09579, + "input_cost": 0.08704, + "output_cost": 0.00875, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "claude-opus-5-tiered_input_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f7bec63ac6ff summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer f7bec63ac6ff" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 204800, + "output_tokens": 620 + } + } + }, + "expected": { + "spend": 2.07125, + "input_cost": 2.048, + "output_cost": 0.02325, + "prompt_tokens": 204800, + "completion_tokens": 620 + } + }, + { + "name": "claude-opus-5-tiered_cache_read_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "58ab3f8e01f6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer 58ab3f8e01f6" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 4096, + "output_tokens": 480, + "cache_read_input_tokens": 201728 + } + } + }, + "expected": { + "spend": 0.260688, + "input_cost": 0.242688, + "output_cost": 0.018, + "prompt_tokens": 205824, + "completion_tokens": 480 + } + }, + { + "name": "claude-opus-5-tiered_cache_write_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1a923968b132 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer 1a923968b132" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 4096, + "output_tokens": 480, + "cache_creation_input_tokens": 200704, + "cache_creation": { + "ephemeral_5m_input_tokens": 200704, + "ephemeral_1h_input_tokens": 0 + } + } + } + }, + "expected": { + "spend": 2.56776, + "input_cost": 2.54976, + "output_cost": 0.018, + "prompt_tokens": 204800, + "completion_tokens": 480 + } + }, + { + "name": "claude-opus-5-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "928b583c6a13 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer 928b583c6a13" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "service_tier": "priority" + } + } + }, + "expected": { + "spend": 0.024375, + "input_cost": 0.0115, + "output_cost": 0.012875, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-anthropic_fast_mode", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "dd7504ab4a95 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer dd7504ab4a95" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "speed": "fast" + } + } + }, + "expected": { + "spend": 0.117, + "input_cost": 0.0552, + "output_cost": 0.0618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-anthropic_us_inference", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7dcf31884733 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer 7dcf31884733" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "inference_geo": "us" + } + } + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e63cb0e28801 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "text", + "text": "scripted answer e63cb0e28801" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": { + "web_search_requests": 3 + } + } + } + }, + "expected": { + "spend": 0.0495, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1fcb7b21debc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 1fcb7b21debc\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e5bff69088af summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer e5bff69088af\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05 + } + } + }, + { + "name": "claude-opus-5-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b6ef7189d74f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"tool_use\", \"id\": \"toolu_$REQUEST_ID\", \"name\": \"get_weather\", \"input\": {}}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"tool_use\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05 + } + } + }, + { + "name": "claude-opus-5-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9901e704cc69 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 9901e704cc69\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5e-06, + "output_cost_per_token": 2.5e-05 + } + } + }, + { + "name": "claude-opus-5-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "14075d9902ec summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer 14075d9902ec" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "aa3357727723 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer aa3357727723\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8e5f37db0dfc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [ + { + "type": "tool_use", + "id": "toolu_$REQUEST_ID", + "name": "get_weather", + "input": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + ], + "stop_reason": "tool_use", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7cfe98295218 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"tool_use\", \"id\": \"toolu_$REQUEST_ID\", \"name\": \"get_weather\", \"input\": {}}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"tool_use\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0195, + "input_cost": 0.0092, + "output_cost": 0.0103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-opus-5-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-opus-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0bacb827a61a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-opus-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840, \"cache_read_input_tokens\": 6144, \"cache_creation_input_tokens\": 3072, \"cache_creation\": {\"ephemeral_5m_input_tokens\": 2048, \"ephemeral_1h_input_tokens\": 1024}}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 0bacb827a61a\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.045612, + "input_cost": 0.035312, + "output_cost": 0.0103, + "prompt_tokens": 11056, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e672859760ae summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer e672859760ae" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "68925ddd50c0 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer 68925ddd50c0" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 640, + "output_tokens": 380, + "cache_read_input_tokens": 12288 + } + } + }, + "expected": { + "spend": 0.0113064, + "input_cost": 0.0056064, + "output_cost": 0.0057, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "claude-sonnet-5-cache_write_5m", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "212f38c1ea0d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer 212f38c1ea0d" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 9216, + "ephemeral_1h_input_tokens": 0 + } + } + } + }, + "expected": { + "spend": 0.041346, + "input_cost": 0.036096, + "output_cost": 0.00525, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "claude-sonnet-5-cache_write_1h", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "638e0a865af7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer 638e0a865af7" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 512, + "output_tokens": 350, + "cache_creation_input_tokens": 9216, + "cache_creation": { + "ephemeral_5m_input_tokens": 2048, + "ephemeral_1h_input_tokens": 7168 + } + } + } + }, + "expected": { + "spend": 0.057474, + "input_cost": 0.052224, + "output_cost": 0.00525, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "claude-sonnet-5-tiered_input_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5ccef99d1220 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer 5ccef99d1220" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 204800, + "output_tokens": 620 + } + } + }, + "expected": { + "spend": 1.24275, + "input_cost": 1.2288, + "output_cost": 0.01395, + "prompt_tokens": 204800, + "completion_tokens": 620 + } + }, + { + "name": "claude-sonnet-5-tiered_cache_read_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "aaa479b1e950 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer aaa479b1e950" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 4096, + "output_tokens": 480, + "cache_read_input_tokens": 201728 + } + } + }, + "expected": { + "spend": 0.1564128, + "input_cost": 0.1456128, + "output_cost": 0.0108, + "prompt_tokens": 205824, + "completion_tokens": 480 + } + }, + { + "name": "claude-sonnet-5-tiered_cache_write_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f3c0e1d4dedd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer f3c0e1d4dedd" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 4096, + "output_tokens": 480, + "cache_creation_input_tokens": 200704, + "cache_creation": { + "ephemeral_5m_input_tokens": 200704, + "ephemeral_1h_input_tokens": 0 + } + } + } + }, + "expected": { + "spend": 1.540656, + "input_cost": 1.529856, + "output_cost": 0.0108, + "prompt_tokens": 204800, + "completion_tokens": 480 + } + }, + { + "name": "claude-sonnet-5-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9863908ec91f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer 9863908ec91f" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "service_tier": "priority" + } + } + }, + "expected": { + "spend": 0.014625, + "input_cost": 0.0069, + "output_cost": 0.007725, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-anthropic_us_inference", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "bb37086ce8e6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer bb37086ce8e6" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "inference_geo": "us" + } + } + }, + "expected": { + "spend": 0.01287, + "input_cost": 0.006072, + "output_cost": 0.006798, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ddcbec1b7eb2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 5 + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "text", + "text": "scripted answer ddcbec1b7eb2" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": { + "web_search_requests": 3 + } + } + } + }, + "expected": { + "spend": 0.0417, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ca259a6916f2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer ca259a6916f2\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b14b060d38cc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer b14b060d38cc\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05 + } + } + }, + { + "name": "claude-sonnet-5-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "23d6e2f6eb94 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"tool_use\", \"id\": \"toolu_$REQUEST_ID\", \"name\": \"get_weather\", \"input\": {}}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"tool_use\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05 + } + } + }, + { + "name": "claude-sonnet-5-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f803710311e5 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer f803710311e5\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05 + } + } + }, + { + "name": "claude-sonnet-5-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "04c8cd550f99 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [ + { + "type": "text", + "text": "scripted answer 04c8cd550f99" + } + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3ca6439c3e0d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-haiku-4-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 3ca6439c3e0d\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0039, + "input_cost": 0.00184, + "output_cost": 0.00206, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e32fe8463152 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "tool_use", + "id": "toolu_$REQUEST_ID", + "name": "get_weather", + "input": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + ], + "stop_reason": "tool_use", + "usage": { + "input_tokens": 1840, + "output_tokens": 412 + } + } + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "11512728994f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"tool_use\", \"id\": \"toolu_$REQUEST_ID\", \"name\": \"get_weather\", \"input\": {}}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"input_json_delta\", \"partial_json\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"tool_use\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0117, + "input_cost": 0.00552, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "claude-sonnet-5-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "claude-sonnet-5", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "543a97cebc29 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: message_start\ndata: {\"type\": \"message_start\", \"message\": {\"id\": \"msg_$REQUEST_ID\", \"type\": \"message\", \"role\": \"assistant\", \"model\": \"claude-sonnet-5\", \"content\": [], \"stop_reason\": null, \"usage\": {\"input_tokens\": 1840, \"cache_read_input_tokens\": 6144, \"cache_creation_input_tokens\": 3072, \"cache_creation\": {\"ephemeral_5m_input_tokens\": 2048, \"ephemeral_1h_input_tokens\": 1024}}}}", + "event: content_block_start\ndata: {\"type\": \"content_block_start\", \"index\": 0, \"content_block\": {\"type\": \"text\", \"text\": \"\"}}", + "event: content_block_delta\ndata: {\"type\": \"content_block_delta\", \"index\": 0, \"delta\": {\"type\": \"text_delta\", \"text\": \"scripted answer 543a97cebc29\"}}", + "event: content_block_stop\ndata: {\"type\": \"content_block_stop\", \"index\": 0}", + "event: message_delta\ndata: {\"type\": \"message_delta\", \"delta\": {\"stop_reason\": \"end_turn\"}, \"usage\": {\"output_tokens\": 412}}", + "event: message_stop\ndata: {\"type\": \"message_stop\"}" + ] + }, + "expected": { + "spend": 0.0273672, + "input_cost": 0.0211872, + "output_cost": 0.00618, + "prompt_tokens": 11056, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "10dc41a37bf4 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788233, + "model": "accounts/fireworks/models/deepseek-v4p1-flash", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 10dc41a37bf4" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-fallback_cache_read_at_input_rate", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ee35d47aaab5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788234, + "model": "accounts/fireworks/models/deepseek-v4p1-flash", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer ee35d47aaab5" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.0021672, + "input_cost": 0.0019392, + "output_cost": 0.000228, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f77cb314f5aa summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer f77cb314f5aa\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c5fac079eac8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer c5fac079eac8\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "eea156c013c8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "124287c4bcaa summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788240, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788240, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 124287c4bcaa\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788240, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4f7445b95bbd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788242, + "model": "accounts/fireworks/models/kimi-k3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 4f7445b95bbd" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8e888a093f6c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788245, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788245, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 8e888a093f6c\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788245, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788245, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ef4a0046af51 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788246, + "model": "accounts/fireworks/models/deepseek-v4p1-flash", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5ae7b84f3854 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788248, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-deepseek-v4p1-flash-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "19d356ecf08f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788251, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788251, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 19d356ecf08f\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788251, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788251, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a9341cd5b3ec summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788229, + "model": "accounts/fireworks/models/kimi-k3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer a9341cd5b3ec" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f80f2a5e5bec summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788229, + "model": "accounts/fireworks/models/kimi-k3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer f80f2a5e5bec" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.00207128, + "input_cost": 0.00112128, + "output_cost": 0.00095, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a764db4a4844 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer a764db4a4844\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f168dea08a8c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788232, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788232, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer f168dea08a8c\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788232, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5f4b9e007f6c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788235, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788235, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788235, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788235, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788235, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788235, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c5d6768437a1 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer c5d6768437a1\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788236, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.5e-06 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e88240789ba9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788236, + "model": "accounts/fireworks/models/qwen3p8-max", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer e88240789ba9" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "454606d6e5ae summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788239, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788239, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 454606d6e5ae\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788239, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788239, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6655aac8edcd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788240, + "model": "accounts/fireworks/models/kimi-k3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cbcf2fb047bb summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788242, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.002134, + "input_cost": 0.001104, + "output_cost": 0.00103, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-kimi-k3-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/kimi-k3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d540b1082db1 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788244, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788244, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer d540b1082db1\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788244, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788244, \"model\": \"accounts/fireworks/models/kimi-k3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 7984, \"completion_tokens\": 412, \"total_tokens\": 8396, \"prompt_tokens_details\": {\"cached_tokens\": 6144}}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.00250264, + "input_cost": 0.00147264, + "output_cost": 0.00103, + "prompt_tokens": 7984, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "037102bc4f02 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788246, + "model": "accounts/fireworks/models/qwen3p8-max", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 037102bc4f02" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "246ab713a447 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788248, + "model": "accounts/fireworks/models/qwen3p8-max", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 246ab713a447" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.00304992, + "input_cost": 0.00168192, + "output_cost": 0.001368, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7fa6702c872d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788250, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788250, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 7fa6702c872d\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788250, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788250, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a52571ae25d8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788228, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788228, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer a52571ae25d8\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788228, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 3.6e-06 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d621057b8000 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788229, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788229, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788229, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788229, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788229, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788229, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 3.6e-06 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e5bcee3da31d summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer e5bcee3da31d\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788231, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 9e-07, + "output_cost_per_token": 3.6e-06 + } + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "eeadd4cae922 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788233, + "model": "accounts/fireworks/models/deepseek-v4p1-flash", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer eeadd4cae922" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8ed9cad57fc4 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788234, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788234, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 8ed9cad57fc4\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788234, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788234, \"model\": \"accounts/fireworks/models/deepseek-v4p1-flash\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0005232, + "input_cost": 0.000276, + "output_cost": 0.0002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "96d301af3055 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788236, + "model": "accounts/fireworks/models/qwen3p8-max", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "389fe82a3e30 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788238, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0031392, + "input_cost": 0.001656, + "output_cost": 0.0014832, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "fireworks_ai-accounts-fireworks-models-qwen3p8-max-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "fireworks_ai/accounts/fireworks/models/qwen3p8-max", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d3a53c5889e6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788241, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788241, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer d3a53c5889e6\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788241, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788241, \"model\": \"accounts/fireworks/models/qwen3p8-max\", \"choices\": [], \"usage\": {\"prompt_tokens\": 7984, \"completion_tokens\": 412, \"total_tokens\": 8396, \"prompt_tokens_details\": {\"cached_tokens\": 6144}}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.00369216, + "input_cost": 0.00220896, + "output_cost": 0.0014832, + "prompt_tokens": 7984, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5fdf6b7dd9b9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 5fdf6b7dd9b9" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "bb9b95a5e878 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer bb9b95a5e878" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 12928, + "candidatesTokenCount": 380, + "totalTokenCount": 13308, + "cachedContentTokenCount": 12288, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 12928 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.00871248, + "input_cost": 0.00392448, + "output_cost": 0.004788, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gemini-3.1-pro-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "eccb8318be2d summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer eccb8318be2d" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1546, + "candidatesTokenCount": 210, + "totalTokenCount": 1756, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 96 + }, + { + "modality": "AUDIO", + "tokenCount": 1450 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0067626, + "input_cost": 0.0041166, + "output_cost": 0.002646, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "gemini-3.1-pro-image_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f50f723a74f1 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer f50f723a74f1" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 2116, + "candidatesTokenCount": 240, + "totalTokenCount": 2356, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 310 + }, + { + "modality": "IMAGE", + "tokenCount": 1806 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0078288, + "input_cost": 0.0048048, + "output_cost": 0.003024, + "prompt_tokens": 2116, + "completion_tokens": 240 + } + }, + { + "name": "gemini-3.1-pro-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a09586282605 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer a09586282605" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1240, + "candidatesTokenCount": 560, + "thoughtsTokenCount": 3480, + "totalTokenCount": 5280, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1240 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.05664, + "input_cost": 0.002604, + "output_cost": 0.054036, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gemini-3.1-pro-tiered_input_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7972fad2f18c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 7972fad2f18c" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 204800, + "candidatesTokenCount": 620, + "totalTokenCount": 205420, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 204800 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.871878, + "input_cost": 0.86016, + "output_cost": 0.011718, + "prompt_tokens": 204800, + "completion_tokens": 620 + } + }, + { + "name": "gemini-3.1-pro-tiered_cache_read_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c6ae4eac7c46 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer c6ae4eac7c46" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 205824, + "candidatesTokenCount": 480, + "totalTokenCount": 206304, + "cachedContentTokenCount": 201728, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 205824 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.11100096, + "input_cost": 0.10192896, + "output_cost": 0.009072, + "prompt_tokens": 205824, + "completion_tokens": 480 + } + }, + { + "name": "gemini-3.1-pro-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "afc20048852d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer afc20048852d" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_FLEX" + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0045276, + "input_cost": 0.001932, + "output_cost": 0.0025956, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c45311f260f5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer c45311f260f5" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_PRIORITY" + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.011319, + "input_cost": 0.00483, + "output_cost": 0.006489, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "73631ea17d2b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleSearch": {} + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 73631ea17d2b" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "query 0", + "query 1", + "query 2" + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.1140552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-google_maps_grounding", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5c6ab7918d5a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleMaps": {} + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 5c6ab7918d5a" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "maps query 0" + ], + "groundingChunks": [ + { + "maps": { + "uri": "https://maps.google.com/?cid=0" + } + } + ], + "googleMapsWidgetContextToken": "token_$REQUEST_ID" + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0340552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-fallback_video_tokens_at_input_rate", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5fc254e9189f summarize the attached material in one line and name the city weather" + }, + { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAAGGZ0eXBpc29tAAACAGlzb21pc282AAACCG1kYXQNFBsiKTA3PkVMU1phaG92fYSLkpmgp661vMPK0djf5u30+wIJEBceJSwzOkFIT1ZdZGtyeYCHjpWco6qxuL/GzdTb4unw9/4FDBMaISgvNj1ES1JZYGdudXyDipGYn6attLvCydDX3uXs8/oBCA8WHSQrMjlAR05VXGNqcXh/ho2Um6KpsLe+xczT2uHo7/b9BAsSGSAnLjU8Q0pRWF9mbXR7gomQl56lrLO6wcjP1t3k6/L5AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK+2vcTL0tng5+71/AMKERgfJi00O0JJUFdeZWxzeoGIj5adpKuyucDHztXc4+rx+P8GDRQbIikwNz5FTFNaYWhvdn2Ei5KZoKeutbzDytHY3+bt9PsCCRAXHiUsMzpBSE9WXWRrcnmAh46VnKOqsbi/xs3U2+Lp8Pf+BQwTGiEoLzY9REtSWWBnbnV8g4qRmJ+mrbS7wsnQ197l7PP6AQgPFh0kKzI5QEdOVVxjanF4f4aNlJuiqbC3vsXM09rh6O/2/QQLEhkgJy41PENKUVhfZm10e4KJkJeepayzusHIz9bd5Ovy+QAHDhUcIyoxOD9GTVRbYmlwd36FjJOaoaivtr3Ey9LZ4Ofu9fwDChEYHyYtNDtCSVBXXmVsc3qBiI+WnaSrsrnAx87V3OPq8fj/Bg==", + "format": "mp4" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 5fc254e9189f" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 8060, + "candidatesTokenCount": 300, + "totalTokenCount": 8360, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 140 + }, + { + "modality": "VIDEO", + "tokenCount": 7920 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.020706, + "input_cost": 0.016926, + "output_cost": 0.00378, + "prompt_tokens": 8060, + "completion_tokens": 300 + } + }, + { + "name": "gemini-3.1-pro-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "52b6a80ff038 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 52b6a80ff038\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4985d6423ec4 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 4985d6423ec4\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2.1e-06, + "output_cost_per_token": 1.26e-05 + } + } + }, + { + "name": "gemini-3.1-pro-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "11bca0892f81 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2.1e-06, + "output_cost_per_token": 1.26e-05 + } + } + }, + { + "name": "gemini-3.1-pro-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4c92224d4b84 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 4c92224d4b84\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2.1e-06, + "output_cost_per_token": 1.26e-05 + } + } + }, + { + "name": "gemini-3.1-pro-prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6c159519a099 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "promptFeedback": { + "blockReason": "SAFETY", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "HIGH", + "blocked": true + } + ] + }, + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 0, + "totalTokenCount": 1840, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.003864, + "input_cost": 0.003864, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-3.1-pro-stream_prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7df769816861 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"promptFeedback\": {\"blockReason\": \"SAFETY\", \"safetyRatings\": [{\"category\": \"HARM_CATEGORY_HARASSMENT\", \"probability\": \"HIGH\", \"blocked\": true}]}, \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 0, \"totalTokenCount\": 1840, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.003864, + "input_cost": 0.003864, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-3.1-pro-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "956e05125691 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 956e05125691" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2048ef936293 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 2048ef936293\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2a68816dc8ea summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7ba6668f10df summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.1-pro-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "54e6d8c321ef summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 54e6d8c321ef\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 8314, \"candidatesTokenCount\": 412, \"thoughtsTokenCount\": 900, \"totalTokenCount\": 9626, \"cachedContentTokenCount\": 6144, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 7984}, {\"modality\": \"AUDIO\", \"tokenCount\": 330}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.02338644, + "input_cost": 0.00604524, + "output_cost": 0.0173412, + "prompt_tokens": 8314, + "completion_tokens": 1312 + } + }, + { + "name": "gemini-3.8-flash-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e7c21c357fb0 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer e7c21c357fb0" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c58ea8fe6a99 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer c58ea8fe6a99" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 12928, + "candidatesTokenCount": 380, + "totalTokenCount": 13308, + "cachedContentTokenCount": 12288, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 12928 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.002157376, + "input_cost": 0.000971776, + "output_cost": 0.0011856, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gemini-3.8-flash-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3e33892c4f9f summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 3e33892c4f9f" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1546, + "candidatesTokenCount": 210, + "totalTokenCount": 1756, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 96 + }, + { + "modality": "AUDIO", + "tokenCount": 1450 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00221312, + "input_cost": 0.00155792, + "output_cost": 0.0006552, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "gemini-3.8-flash-audio_output", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "30bf1c0de6fe summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "modalities": [ + "text", + "audio" + ], + "audio": { + "voice": "alloy", + "format": "pcm16" + }, + "allowed_openai_params": [ + "modalities", + "audio" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 30bf1c0de6fe" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 220, + "candidatesTokenCount": 1300, + "totalTokenCount": 1520, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 220 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 180 + }, + { + "modality": "AUDIO", + "tokenCount": 1120 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0076648, + "input_cost": 0.0001144, + "output_cost": 0.0075504, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + }, + { + "name": "gemini-3.8-flash-video_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5f5da5957185 summarize the attached material in one line and name the city weather" + }, + { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAAGGZ0eXBpc29tAAACAGlzb21pc282AAACCG1kYXQNFBsiKTA3PkVMU1phaG92fYSLkpmgp661vMPK0djf5u30+wIJEBceJSwzOkFIT1ZdZGtyeYCHjpWco6qxuL/GzdTb4unw9/4FDBMaISgvNj1ES1JZYGdudXyDipGYn6attLvCydDX3uXs8/oBCA8WHSQrMjlAR05VXGNqcXh/ho2Um6KpsLe+xczT2uHo7/b9BAsSGSAnLjU8Q0pRWF9mbXR7gomQl56lrLO6wcjP1t3k6/L5AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK+2vcTL0tng5+71/AMKERgfJi00O0JJUFdeZWxzeoGIj5adpKuyucDHztXc4+rx+P8GDRQbIikwNz5FTFNaYWhvdn2Ei5KZoKeutbzDytHY3+bt9PsCCRAXHiUsMzpBSE9WXWRrcnmAh46VnKOqsbi/xs3U2+Lp8Pf+BQwTGiEoLzY9REtSWWBnbnV8g4qRmJ+mrbS7wsnQ197l7PP6AQgPFh0kKzI5QEdOVVxjanF4f4aNlJuiqbC3vsXM09rh6O/2/QQLEhkgJy41PENKUVhfZm10e4KJkJeepayzusHIz9bd5Ovy+QAHDhUcIyoxOD9GTVRbYmlwd36FjJOaoaivtr3Ey9LZ4Ofu9fwDChEYHyYtNDtCSVBXXmVsc3qBiI+WnaSrsrnAx87V3OPq8fj/Bg==", + "format": "mp4" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 5f5da5957185" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 8060, + "candidatesTokenCount": 300, + "totalTokenCount": 8360, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 140 + }, + { + "modality": "VIDEO", + "tokenCount": 7920 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0059192, + "input_cost": 0.0049832, + "output_cost": 0.000936, + "prompt_tokens": 8060, + "completion_tokens": 300 + } + }, + { + "name": "gemini-3.8-flash-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0009f5ac891e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 0009f5ac891e" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_FLEX" + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00112112, + "input_cost": 0.0004784, + "output_cost": 0.00064272, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "057d8b15b597 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 057d8b15b597" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_PRIORITY" + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0028028, + "input_cost": 0.001196, + "output_cost": 0.0016068, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-web_search_per_prompt", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c409248006ff summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleSearch": {} + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer c409248006ff" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "query 0", + "query 1", + "query 2" + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.03724224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-google_maps_grounding", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cdcfe11184ca summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleMaps": {} + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer cdcfe11184ca" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "maps query 0" + ], + "groundingChunks": [ + { + "maps": { + "uri": "https://maps.google.com/?cid=0" + } + } + ], + "googleMapsWidgetContextToken": "token_$REQUEST_ID" + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.02724224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-fallback_reasoning_at_output_rate", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f92946792f44 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer f92946792f44" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1240, + "candidatesTokenCount": 560, + "thoughtsTokenCount": 3480, + "totalTokenCount": 5280, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1240 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0132496, + "input_cost": 0.0006448, + "output_cost": 0.0126048, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gemini-3.8-flash-fallback_image_tokens_at_input_rate", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "59106006ecc4 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 59106006ecc4" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 2116, + "candidatesTokenCount": 240, + "totalTokenCount": 2356, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 310 + }, + { + "modality": "IMAGE", + "tokenCount": 1806 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00184912, + "input_cost": 0.00110032, + "output_cost": 0.0007488, + "prompt_tokens": 2116, + "completion_tokens": 240 + } + }, + { + "name": "gemini-3.8-flash-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "02cc764f4300 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 02cc764f4300\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "60f7b65abfa3 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 60f7b65abfa3\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.2e-07, + "output_cost_per_token": 3.12e-06 + } + } + }, + { + "name": "gemini-3.8-flash-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "18632b64dd03 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.2e-07, + "output_cost_per_token": 3.12e-06 + } + } + }, + { + "name": "gemini-3.8-flash-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a3126f19100d summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer a3126f19100d\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.2e-07, + "output_cost_per_token": 3.12e-06 + } + } + }, + { + "name": "gemini-3.8-flash-prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "899380691bc7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "promptFeedback": { + "blockReason": "SAFETY", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "HIGH", + "blocked": true + } + ] + }, + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 0, + "totalTokenCount": 1840, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0009568, + "input_cost": 0.0009568, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-3.8-flash-stream_prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4c9331e5da39 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"promptFeedback\": {\"blockReason\": \"SAFETY\", \"safetyRatings\": [{\"category\": \"HARM_CATEGORY_HARASSMENT\", \"probability\": \"HIGH\", \"blocked\": true}]}, \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 0, \"totalTokenCount\": 1840, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.0009568, + "input_cost": 0.0009568, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-3.8-flash-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4972a11cd52d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 4972a11cd52d" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0908445fc9e7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 0908445fc9e7\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.0090552, + "input_cost": 0.003864, + "output_cost": 0.0051912, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "151d9709f7f7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9253170bf979 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.00224224, + "input_cost": 0.0009568, + "output_cost": 0.00128544, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-3.8-flash-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e31c97cab9cc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer e31c97cab9cc\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 8314, \"candidatesTokenCount\": 692, \"thoughtsTokenCount\": 900, \"totalTokenCount\": 9906, \"cachedContentTokenCount\": 6144, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 7984}, {\"modality\": \"AUDIO\", \"tokenCount\": 330}], \"candidatesTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 412}, {\"modality\": \"AUDIO\", \"tokenCount\": 280}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.007460128, + "input_cost": 0.001619488, + "output_cost": 0.00584064, + "prompt_tokens": 8314, + "completion_tokens": 1592 + } + }, + { + "name": "gemini-gemini-3.1-pro-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "15e6a9747fd2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 15e6a9747fd2" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "722017e8e394 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 722017e8e394" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 12928, + "candidatesTokenCount": 380, + "totalTokenCount": 13308, + "cachedContentTokenCount": 12288, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 12928 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0082976, + "input_cost": 0.0037376, + "output_cost": 0.00456, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gemini-gemini-3.1-pro-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f03ad3a1bb53 summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer f03ad3a1bb53" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1546, + "candidatesTokenCount": 210, + "totalTokenCount": 1756, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 96 + }, + { + "modality": "AUDIO", + "tokenCount": 1450 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.006482, + "input_cost": 0.003962, + "output_cost": 0.00252, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "gemini-gemini-3.1-pro-image_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0c05c06c97fa summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 0c05c06c97fa" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 2116, + "candidatesTokenCount": 240, + "totalTokenCount": 2356, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 310 + }, + { + "modality": "IMAGE", + "tokenCount": 1806 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.0074732, + "input_cost": 0.0045932, + "output_cost": 0.00288, + "prompt_tokens": 2116, + "completion_tokens": 240 + } + }, + { + "name": "gemini-gemini-3.1-pro-video_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7c49c7c888f4 summarize the attached material in one line and name the city weather" + }, + { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAAGGZ0eXBpc29tAAACAGlzb21pc282AAACCG1kYXQNFBsiKTA3PkVMU1phaG92fYSLkpmgp661vMPK0djf5u30+wIJEBceJSwzOkFIT1ZdZGtyeYCHjpWco6qxuL/GzdTb4unw9/4FDBMaISgvNj1ES1JZYGdudXyDipGYn6attLvCydDX3uXs8/oBCA8WHSQrMjlAR05VXGNqcXh/ho2Um6KpsLe+xczT2uHo7/b9BAsSGSAnLjU8Q0pRWF9mbXR7gomQl56lrLO6wcjP1t3k6/L5AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK+2vcTL0tng5+71/AMKERgfJi00O0JJUFdeZWxzeoGIj5adpKuyucDHztXc4+rx+P8GDRQbIikwNz5FTFNaYWhvdn2Ei5KZoKeutbzDytHY3+bt9PsCCRAXHiUsMzpBSE9WXWRrcnmAh46VnKOqsbi/xs3U2+Lp8Pf+BQwTGiEoLzY9REtSWWBnbnV8g4qRmJ+mrbS7wsnQ197l7PP6AQgPFh0kKzI5QEdOVVxjanF4f4aNlJuiqbC3vsXM09rh6O/2/QQLEhkgJy41PENKUVhfZm10e4KJkJeepayzusHIz9bd5Ovy+QAHDhUcIyoxOD9GTVRbYmlwd36FjJOaoaivtr3Ey9LZ4Ofu9fwDChEYHyYtNDtCSVBXXmVsc3qBiI+WnaSrsrnAx87V3OPq8fj/Bg==", + "format": "mp4" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 7c49c7c888f4" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 8060, + "candidatesTokenCount": 300, + "totalTokenCount": 8360, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 140 + }, + { + "modality": "VIDEO", + "tokenCount": 7920 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.022888, + "input_cost": 0.019288, + "output_cost": 0.0036, + "prompt_tokens": 8060, + "completion_tokens": 300 + } + }, + { + "name": "gemini-gemini-3.1-pro-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "000113942d5d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 000113942d5d" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1240, + "candidatesTokenCount": 560, + "thoughtsTokenCount": 3480, + "totalTokenCount": 5280, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1240 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.05444, + "input_cost": 0.00248, + "output_cost": 0.05196, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gemini-gemini-3.1-pro-tiered_input_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1dc16abc4658 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 1dc16abc4658" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 204800, + "candidatesTokenCount": 620, + "totalTokenCount": 205420, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 204800 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.83036, + "input_cost": 0.8192, + "output_cost": 0.01116, + "prompt_tokens": 204800, + "completion_tokens": 620 + } + }, + { + "name": "gemini-gemini-3.1-pro-tiered_cache_read_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e0ceec272f4b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer e0ceec272f4b" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 205824, + "candidatesTokenCount": 480, + "totalTokenCount": 206304, + "cachedContentTokenCount": 201728, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 205824 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.1057152, + "input_cost": 0.0970752, + "output_cost": 0.00864, + "prompt_tokens": 205824, + "completion_tokens": 480 + } + }, + { + "name": "gemini-gemini-3.1-pro-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "888d93f4c060 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 888d93f4c060" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_FLEX" + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.004312, + "input_cost": 0.00184, + "output_cost": 0.002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "68dfafa41eed summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 68dfafa41eed" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_PRIORITY" + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.01078, + "input_cost": 0.0046, + "output_cost": 0.00618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3d8a4ab5a9b2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleSearch": {} + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 3d8a4ab5a9b2" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "query 0", + "query 1", + "query 2" + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.113624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-google_maps_grounding", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "982de823fd3b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleMaps": {} + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 982de823fd3b" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "maps query 0" + ], + "groundingChunks": [ + { + "maps": { + "uri": "https://maps.google.com/?cid=0" + } + } + ], + "googleMapsWidgetContextToken": "token_$REQUEST_ID" + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.033624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "93682132cbf8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 93682132cbf8\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "661f87e3dcf5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 661f87e3dcf5\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9fc58c44c867 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5aced106bb93 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 5aced106bb93\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gemini-gemini-3.1-pro-prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0b401759be94 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "promptFeedback": { + "blockReason": "SAFETY", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "HIGH", + "blocked": true + } + ] + }, + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 0, + "totalTokenCount": 1840, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.00368, + "input_cost": 0.00368, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "46376a43606c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"promptFeedback\": {\"blockReason\": \"SAFETY\", \"safetyRatings\": [{\"category\": \"HARM_CATEGORY_HARASSMENT\", \"probability\": \"HIGH\", \"blocked\": true}]}, \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 0, \"totalTokenCount\": 1840, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.00368, + "input_cost": 0.00368, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-gemini-3.1-pro-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "da151058cfb9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer da151058cfb9" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d2aaa2ca1279 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer d2aaa2ca1279\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4fe80308a236 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f0ddadf59ebc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.1-pro-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.1-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "47de5dc94825 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 47de5dc94825\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 8314, \"candidatesTokenCount\": 412, \"thoughtsTokenCount\": 900, \"totalTokenCount\": 9626, \"cachedContentTokenCount\": 6144, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 7984}, {\"modality\": \"AUDIO\", \"tokenCount\": 330}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.0224108, + "input_cost": 0.0057668, + "output_cost": 0.016644, + "prompt_tokens": 8314, + "completion_tokens": 1312 + } + }, + { + "name": "gemini-gemini-3.8-flash-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3944829b75e5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 3944829b75e5" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "24a568396212 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 24a568396212" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 12928, + "candidatesTokenCount": 380, + "totalTokenCount": 13308, + "cachedContentTokenCount": 12288, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 12928 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0020744, + "input_cost": 0.0009344, + "output_cost": 0.00114, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gemini-gemini-3.8-flash-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "99e65f16c4b4 summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 99e65f16c4b4" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1546, + "candidatesTokenCount": 210, + "totalTokenCount": 1756, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 96 + }, + { + "modality": "AUDIO", + "tokenCount": 1450 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.002128, + "input_cost": 0.001498, + "output_cost": 0.00063, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "gemini-gemini-3.8-flash-audio_output", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6fc6e4823e02 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "modalities": [ + "text", + "audio" + ], + "audio": { + "voice": "alloy", + "format": "pcm16" + }, + "allowed_openai_params": [ + "modalities", + "audio" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 6fc6e4823e02" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 220, + "candidatesTokenCount": 1300, + "totalTokenCount": 1520, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 220 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 180 + }, + { + "modality": "AUDIO", + "tokenCount": 1120 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00737, + "input_cost": 0.00011, + "output_cost": 0.00726, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + }, + { + "name": "gemini-gemini-3.8-flash-image_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ca377dd90846 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer ca377dd90846" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 2116, + "candidatesTokenCount": 240, + "totalTokenCount": 2356, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 310 + }, + { + "modality": "IMAGE", + "tokenCount": 1806 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.0018683, + "input_cost": 0.0011483, + "output_cost": 0.00072, + "prompt_tokens": 2116, + "completion_tokens": 240 + } + }, + { + "name": "gemini-gemini-3.8-flash-video_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "030071a5c80f summarize the attached material in one line and name the city weather" + }, + { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAAGGZ0eXBpc29tAAACAGlzb21pc282AAACCG1kYXQNFBsiKTA3PkVMU1phaG92fYSLkpmgp661vMPK0djf5u30+wIJEBceJSwzOkFIT1ZdZGtyeYCHjpWco6qxuL/GzdTb4unw9/4FDBMaISgvNj1ES1JZYGdudXyDipGYn6attLvCydDX3uXs8/oBCA8WHSQrMjlAR05VXGNqcXh/ho2Um6KpsLe+xczT2uHo7/b9BAsSGSAnLjU8Q0pRWF9mbXR7gomQl56lrLO6wcjP1t3k6/L5AAcOFRwjKjE4P0ZNVFtiaXB3foWMk5qhqK+2vcTL0tng5+71/AMKERgfJi00O0JJUFdeZWxzeoGIj5adpKuyucDHztXc4+rx+P8GDRQbIikwNz5FTFNaYWhvdn2Ei5KZoKeutbzDytHY3+bt9PsCCRAXHiUsMzpBSE9WXWRrcnmAh46VnKOqsbi/xs3U2+Lp8Pf+BQwTGiEoLzY9REtSWWBnbnV8g4qRmJ+mrbS7wsnQ197l7PP6AQgPFh0kKzI5QEdOVVxjanF4f4aNlJuiqbC3vsXM09rh6O/2/QQLEhkgJy41PENKUVhfZm10e4KJkJeepayzusHIz9bd5Ovy+QAHDhUcIyoxOD9GTVRbYmlwd36FjJOaoaivtr3Ey9LZ4Ofu9fwDChEYHyYtNDtCSVBXXmVsc3qBiI+WnaSrsrnAx87V3OPq8fj/Bg==", + "format": "mp4" + } + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 030071a5c80f" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 8060, + "candidatesTokenCount": 300, + "totalTokenCount": 8360, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 140 + }, + { + "modality": "VIDEO", + "tokenCount": 7920 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.005722, + "input_cost": 0.004822, + "output_cost": 0.0009, + "prompt_tokens": 8060, + "completion_tokens": 300 + } + }, + { + "name": "gemini-gemini-3.8-flash-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "518ba3ee4c33 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 518ba3ee4c33" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1240, + "candidatesTokenCount": 560, + "thoughtsTokenCount": 3480, + "totalTokenCount": 5280, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1240 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.01448, + "input_cost": 0.00062, + "output_cost": 0.01386, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gemini-gemini-3.8-flash-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ed5fc114b878 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer ed5fc114b878" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_FLEX" + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.001078, + "input_cost": 0.00046, + "output_cost": 0.000618, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c8b02e840d2c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer c8b02e840d2c" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ], + "trafficType": "ON_DEMAND_PRIORITY" + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.002695, + "input_cost": 0.00115, + "output_cost": 0.001545, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-web_search_per_prompt", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4c0023a5b762 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleSearch": {} + } + ], + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 4c0023a5b762" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "query 0", + "query 1", + "query 2" + ] + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.037156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-google_maps_grounding", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b8da7a958abf summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "googleMaps": {} + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer b8da7a958abf" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": { + "webSearchQueries": [ + "maps query 0" + ], + "groundingChunks": [ + { + "maps": { + "uri": "https://maps.google.com/?cid=0" + } + } + ], + "googleMapsWidgetContextToken": "token_$REQUEST_ID" + } + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.027156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "eacbb9f405ad summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer eacbb9f405ad\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "837d58c93751 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 837d58c93751\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06 + } + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5d097120da02 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06 + } + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5fca57bc9ae9 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 5fca57bc9ae9\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06 + } + } + }, + { + "name": "gemini-gemini-3.8-flash-prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7dc69adaf49c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "promptFeedback": { + "blockReason": "SAFETY", + "safetyRatings": [ + { + "category": "HARM_CATEGORY_HARASSMENT", + "probability": "HIGH", + "blocked": true + } + ] + }, + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 0, + "totalTokenCount": 1840, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.00092, + "input_cost": 0.00092, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_prompt_blocked", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "23065669b96e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"promptFeedback\": {\"blockReason\": \"SAFETY\", \"safetyRatings\": [{\"category\": \"HARM_CATEGORY_HARASSMENT\", \"probability\": \"HIGH\", \"blocked\": true}]}, \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 0, \"totalTokenCount\": 1840, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.00092, + "input_cost": 0.00092, + "output_cost": 0.0, + "prompt_tokens": 1840, + "completion_tokens": 0 + } + }, + { + "name": "gemini-gemini-3.8-flash-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "290ca0555ee8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "scripted answer 290ca0555ee8" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.1-pro" + } + }, + "expected": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ec61060b88e9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer ec61060b88e9\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.1-pro\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.1-pro\"}" + ] + }, + "expected": { + "spend": 0.008624, + "input_cost": 0.00368, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "670f41936a6d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "index": 0 + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 1840 + } + ] + }, + "modelVersion": "gemini-3.8-flash" + } + }, + "expected": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8cdfceec775e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"functionCall\": {\"name\": \"get_weather\", \"args\": {\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}}}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 1840, \"candidatesTokenCount\": 412, \"totalTokenCount\": 2252, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 1840}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.002156, + "input_cost": 0.00092, + "output_cost": 0.001236, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gemini-gemini-3.8-flash-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gemini/gemini-3.8-flash", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0b11ac8b4a63 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"candidates\": [{\"content\": {\"parts\": [{\"text\": \"scripted answer 0b11ac8b4a63\"}], \"role\": \"model\"}, \"finishReason\": \"STOP\", \"index\": 0}], \"modelVersion\": \"gemini-3.8-flash\"}", + "data: {\"candidates\": [], \"usageMetadata\": {\"promptTokenCount\": 8314, \"candidatesTokenCount\": 692, \"thoughtsTokenCount\": 900, \"totalTokenCount\": 9906, \"cachedContentTokenCount\": 6144, \"promptTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 7984}, {\"modality\": \"AUDIO\", \"tokenCount\": 330}], \"candidatesTokensDetails\": [{\"modality\": \"TEXT\", \"tokenCount\": 412}, {\"modality\": \"AUDIO\", \"tokenCount\": 280}]}, \"modelVersion\": \"gemini-3.8-flash\"}" + ] + }, + "expected": { + "spend": 0.0076232, + "input_cost": 0.0015572, + "output_cost": 0.006066, + "prompt_tokens": 8314, + "completion_tokens": 1592 + } + }, + { + "name": "gpt-5.3-codex-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0bb211ce54ec summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 0bb211ce54ec", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "a1f465df7d59 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer a1f465df7d59", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.0073632, + "input_cost": 0.0028032, + "output_cost": 0.00456, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gpt-5.3-codex-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d47bef2ddfda summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788254, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer d47bef2ddfda", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1240, + "output_tokens": 4040, + "total_tokens": 5280, + "output_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.05382, + "input_cost": 0.00186, + "output_cost": 0.05196, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gpt-5.3-codex-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c6c8dc8b11ca summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788255, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer c6c8dc8b11ca", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.003852, + "input_cost": 0.00138, + "output_cost": 0.002472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "807b82ab682a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788256, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 807b82ab682a", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.015408, + "input_cost": 0.00552, + "output_cost": 0.009888, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7c4724f24131 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "medium" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788256, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "web_search_call", + "id": "ws_0", + "status": "completed" + }, + { + "type": "web_search_call", + "id": "ws_1", + "status": "completed" + }, + { + "type": "web_search_call", + "id": "ws_2", + "status": "completed" + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 7c4724f24131", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.045204, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-web_search_low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "dfd08d79f164 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "low" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788257, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "web_search_call", + "id": "ws_0", + "status": "completed" + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer dfd08d79f164", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.017704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-web_search_high", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "401e61950557 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "high" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "web_search_call", + "id": "ws_0", + "status": "completed" + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 401e61950557", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.022704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-file_search", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8ebc31c05806 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "file_search", + "vector_store_ids": [ + "vs_cost_calc_fixture" + ] + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "file_search_call", + "id": "fs_0", + "status": "completed", + "queries": [ + "query 0" + ], + "results": [] + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 8ebc31c05806", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.010204, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "75b82927ffe4 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788254, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 75b82927ffe4\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 75b82927ffe4\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788254, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 75b82927ffe4\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "430855aa14e3 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 430855aa14e3\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 430855aa14e3\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 430855aa14e3\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gpt-5.3-codex-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5e8dac751b8d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_item.added\ndata: {\"type\": \"response.output_item.added\", \"output_index\": 0, \"item\": {\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"\", \"status\": \"in_progress\"}}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.function_call_arguments.done\ndata: {\"type\": \"response.function_call_arguments.done\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gpt-5.3-codex-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0245ffd5ae0f summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788256, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 0245ffd5ae0f\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 0245ffd5ae0f\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788256, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 0245ffd5ae0f\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gpt-5.3-codex-stream_incomplete", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "15523b94e3fe summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 15523b94e3fe\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 15523b94e3fe\"}", + "event: response.incomplete\ndata: {\"type\": \"response.incomplete\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"status\": \"incomplete\", \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 15523b94e3fe\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream_no_usage_incomplete", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6dcd71cdfaa5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 6dcd71cdfaa5\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 6dcd71cdfaa5\"}", + "event: response.incomplete\ndata: {\"type\": \"response.incomplete\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788255, \"status\": \"incomplete\", \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 6dcd71cdfaa5\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gpt-5.3-codex-stream_unvalidated", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2de88869bcff summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 2de88869bcff\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 1, \"content_index\": 0, \"delta\": \"scripted answer 2de88869bcff\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 2de88869bcff\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream_no_usage_unvalidated", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "da22f5aa5869 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer da22f5aa5869\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 1, \"content_index\": 0, \"delta\": \"scripted answer da22f5aa5869\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer da22f5aa5869\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.2e-05 + } + } + }, + { + "name": "gpt-5.3-codex-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5210d175a94f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788258, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 5210d175a94f", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b7edc51cdfbe summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788258, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer b7edc51cdfbe\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer b7edc51cdfbe\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788258, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer b7edc51cdfbe\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "6bf8aad8967c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788257, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "function_call", + "id": "fc_$REQUEST_ID", + "call_id": "call_$REQUEST_ID", + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}", + "status": "completed" + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "01f141cd9d3b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788258, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_item.added\ndata: {\"type\": \"response.output_item.added\", \"output_index\": 0, \"item\": {\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"\", \"status\": \"in_progress\"}}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.function_call_arguments.done\ndata: {\"type\": \"response.function_call_arguments.done\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788258, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.3-codex-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.3-codex", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8ef8970518fd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788258, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 8ef8970518fd\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 8ef8970518fd\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788258, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 8ef8970518fd\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 7984, \"output_tokens\": 1312, \"total_tokens\": 9296, \"input_tokens_details\": {\"cached_tokens\": 6144}, \"output_tokens_details\": {\"reasoning_tokens\": 900}}}}" + ] + }, + "expected": { + "spend": 0.0203256, + "input_cost": 0.0036816, + "output_cost": 0.016644, + "prompt_tokens": 7984, + "completion_tokens": 1312 + } + }, + { + "name": "gpt-5.4-mini-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1157fc293d72 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788259, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 1157fc293d72" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "918d015fad34 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788260, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 918d015fad34" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.00171808, + "input_cost": 0.00065408, + "output_cost": 0.001064, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gpt-5.4-mini-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "b1238d45e42d summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788261, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer b1238d45e42d" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1546, + "completion_tokens": 210, + "total_tokens": 1756, + "prompt_tokens_details": { + "audio_tokens": 1450 + } + } + } + }, + "expected": { + "spend": 0.0151216, + "input_cost": 0.0145336, + "output_cost": 0.000588, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "gpt-5.4-mini-audio_output", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "09073c011cb2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "modalities": [ + "text", + "audio" + ], + "audio": { + "voice": "alloy", + "format": "pcm16" + }, + "allowed_openai_params": [ + "modalities", + "audio" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788257, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 09073c011cb2" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 220, + "completion_tokens": 1300, + "total_tokens": 1520, + "completion_tokens_details": { + "audio_tokens": 1120 + } + } + } + }, + "expected": { + "spend": 0.022981, + "input_cost": 7.7e-05, + "output_cost": 0.022904, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + }, + { + "name": "gpt-5.4-mini-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cfc4c1747119 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788258, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer cfc4c1747119" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1240, + "completion_tokens": 4040, + "total_tokens": 5280, + "completion_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.013138, + "input_cost": 0.000434, + "output_cost": 0.012704, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gpt-5.4-mini-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9bb4305a36a5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788259, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 9bb4305a36a5" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "flex" + } + }, + "expected": { + "spend": 0.0008988, + "input_cost": 0.000322, + "output_cost": 0.0005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4ebd6b6e27b7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788260, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 4ebd6b6e27b7" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "priority" + } + }, + "expected": { + "spend": 0.0035952, + "input_cost": 0.001288, + "output_cost": 0.0023072, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "222c74ef3df5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "medium" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788261, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 222c74ef3df5", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0142976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-web_search_low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "75b4bbcdb164 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "low" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788258, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 75b4bbcdb164", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0117976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-web_search_high", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f2ded281685d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "high" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788259, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer f2ded281685d", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0167976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "85a0f6230523 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788260, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788260, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 85a0f6230523\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788260, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788260, \"model\": \"gpt-5.4-mini\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8e85cbc8b78c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 8e85cbc8b78c\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 2.8e-06 + } + } + }, + { + "name": "gpt-5.4-mini-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "24414e14870e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 2.8e-06 + } + } + }, + { + "name": "gpt-5.4-mini-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ddb683a1724a summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer ddb683a1724a\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 3.5e-07, + "output_cost_per_token": 2.8e-06 + } + } + }, + { + "name": "gpt-5.4-mini-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "dbcf34530ce5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788258, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer dbcf34530ce5" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ec3873b5f576 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788259, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788259, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer ec3873b5f576\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788259, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788259, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "454b9573dcf5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788260, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c3e8188e02bf summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788261, \"model\": \"gpt-5.4-mini\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.4-mini-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.4-mini", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3efb75339951 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 3efb75339951\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788258, \"model\": \"gpt-5.4-mini\", \"choices\": [], \"usage\": {\"prompt_tokens\": 8314, \"completion_tokens\": 1592, \"total_tokens\": 9906, \"prompt_tokens_details\": {\"cached_tokens\": 6144, \"audio_tokens\": 330}, \"completion_tokens_details\": {\"reasoning_tokens\": 900, \"audio_tokens\": 280}}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.01379264, + "input_cost": 0.00415904, + "output_cost": 0.0096336, + "prompt_tokens": 8314, + "completion_tokens": 1592 + } + }, + { + "name": "gpt-5.5-pro-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "eef4c5fe3dab summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788259, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer eef4c5fe3dab", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f6fd81220aad summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788260, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer f6fd81220aad", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 12928, + "output_tokens": 380, + "total_tokens": 13308, + "input_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.073632, + "input_cost": 0.028032, + "output_cost": 0.0456, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gpt-5.5-pro-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "09757dcdc501 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788260, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 09757dcdc501", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1240, + "output_tokens": 4040, + "total_tokens": 5280, + "output_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.5382, + "input_cost": 0.0186, + "output_cost": 0.5196, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gpt-5.5-pro-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e21acaffe79b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788258, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer e21acaffe79b", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.03852, + "input_cost": 0.0138, + "output_cost": 0.02472, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7fee6f8e184f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788259, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 7fee6f8e184f", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.15408, + "input_cost": 0.0552, + "output_cost": 0.09888, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d04d4797f3d0 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "medium" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788260, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "web_search_call", + "id": "ws_0", + "status": "completed" + }, + { + "type": "web_search_call", + "id": "ws_1", + "status": "completed" + }, + { + "type": "web_search_call", + "id": "ws_2", + "status": "completed" + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer d04d4797f3d0", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.11454, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-web_search_low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3ce53f3d07ab summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "low" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788261, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "web_search_call", + "id": "ws_0", + "status": "completed" + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 3ce53f3d07ab", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.08704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-web_search_high", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d084299afdbf summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "high" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788259, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "web_search_call", + "id": "ws_0", + "status": "completed" + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer d084299afdbf", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.09204, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-file_search", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0720f466abdc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "file_search", + "vector_store_ids": [ + "vs_cost_calc_fixture" + ] + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788260, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "file_search_call", + "id": "fs_0", + "status": "completed", + "queries": [ + "query 0" + ], + "results": [] + }, + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer 0720f466abdc", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.07954, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1130d4d6e2dc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788260, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 1130d4d6e2dc\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 1130d4d6e2dc\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788260, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 1130d4d6e2dc\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "62836f5d3fa3 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 62836f5d3fa3\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 62836f5d3fa3\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 62836f5d3fa3\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012 + } + } + }, + { + "name": "gpt-5.5-pro-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "12478a1a276d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788260, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_item.added\ndata: {\"type\": \"response.output_item.added\", \"output_index\": 0, \"item\": {\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"\", \"status\": \"in_progress\"}}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.function_call_arguments.done\ndata: {\"type\": \"response.function_call_arguments.done\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788260, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012 + } + } + }, + { + "name": "gpt-5.5-pro-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "be189bbbfebe summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer be189bbbfebe\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer be189bbbfebe\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer be189bbbfebe\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012 + } + } + }, + { + "name": "gpt-5.5-pro-stream_incomplete", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "cad50498b33a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer cad50498b33a\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer cad50498b33a\"}", + "event: response.incomplete\ndata: {\"type\": \"response.incomplete\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"status\": \"incomplete\", \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer cad50498b33a\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream_no_usage_incomplete", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2f6f49c3d0f2 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788262, \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 2f6f49c3d0f2\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 2f6f49c3d0f2\"}", + "event: response.incomplete\ndata: {\"type\": \"response.incomplete\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788262, \"status\": \"incomplete\", \"incomplete_details\": {\"reason\": \"max_output_tokens\"}, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 2f6f49c3d0f2\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012 + } + } + }, + { + "name": "gpt-5.5-pro-stream_unvalidated", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "9da2a01340b8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 9da2a01340b8\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 1, \"content_index\": 0, \"delta\": \"scripted answer 9da2a01340b8\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 9da2a01340b8\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream_no_usage_unvalidated", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d162da290b52 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer d162da290b52\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 1, \"content_index\": 0, \"delta\": \"scripted answer d162da290b52\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": \"not-a-number\", \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"scripted_future_item\", \"id\": \"fut_$REQUEST_ID\", \"status\": \"completed\"}, {\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer d162da290b52\", \"annotations\": []}]}]}}" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.5e-05, + "output_cost_per_token": 0.00012 + } + } + }, + { + "name": "gpt-5.5-pro-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d307e0210e1e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788262, + "status": "completed", + "model": "gpt-5.3-codex", + "output": [ + { + "type": "message", + "id": "msg_$REQUEST_ID", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "scripted answer d307e0210e1e", + "annotations": [] + } + ] + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "897338ee89fc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 897338ee89fc\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer 897338ee89fc\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788261, \"status\": \"completed\", \"model\": \"gpt-5.3-codex\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer 897338ee89fc\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.007704, + "input_cost": 0.00276, + "output_cost": 0.004944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "17d97c0f8b6e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "resp_$REQUEST_ID", + "object": "response", + "created_at": 1789788262, + "status": "completed", + "model": "gpt-5.5-pro", + "output": [ + { + "type": "function_call", + "id": "fc_$REQUEST_ID", + "call_id": "call_$REQUEST_ID", + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}", + "status": "completed" + } + ], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "369677236d4b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788263, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_item.added\ndata: {\"type\": \"response.output_item.added\", \"output_index\": 0, \"item\": {\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"\", \"status\": \"in_progress\"}}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}", + "event: response.function_call_arguments.delta\ndata: {\"type\": \"response.function_call_arguments.delta\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"delta\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.function_call_arguments.done\ndata: {\"type\": \"response.function_call_arguments.done\", \"item_id\": \"fc_$REQUEST_ID\", \"output_index\": 0, \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788263, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"function_call\", \"id\": \"fc_$REQUEST_ID\", \"call_id\": \"call_$REQUEST_ID\", \"name\": \"get_weather\", \"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\", \"status\": \"completed\"}], \"usage\": {\"input_tokens\": 1840, \"output_tokens\": 412, \"total_tokens\": 2252}}}" + ] + }, + "expected": { + "spend": 0.07704, + "input_cost": 0.0276, + "output_cost": 0.04944, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.5-pro-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.5-pro", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c9d9cd92af28 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "event: response.created\ndata: {\"type\": \"response.created\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788262, \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer c9d9cd92af28\", \"annotations\": []}]}], \"status\": \"in_progress\", \"usage\": null}}", + "event: response.output_text.delta\ndata: {\"type\": \"response.output_text.delta\", \"item_id\": \"msg_$REQUEST_ID\", \"output_index\": 0, \"content_index\": 0, \"delta\": \"scripted answer c9d9cd92af28\"}", + "event: response.completed\ndata: {\"type\": \"response.completed\", \"response\": {\"id\": \"resp_$REQUEST_ID\", \"object\": \"response\", \"created_at\": 1789788262, \"status\": \"completed\", \"model\": \"gpt-5.5-pro\", \"output\": [{\"type\": \"message\", \"id\": \"msg_$REQUEST_ID\", \"status\": \"completed\", \"role\": \"assistant\", \"content\": [{\"type\": \"output_text\", \"text\": \"scripted answer c9d9cd92af28\", \"annotations\": []}]}], \"usage\": {\"input_tokens\": 7984, \"output_tokens\": 1312, \"total_tokens\": 9296, \"input_tokens_details\": {\"cached_tokens\": 6144}, \"output_tokens_details\": {\"reasoning_tokens\": 900}}}}" + ] + }, + "expected": { + "spend": 0.203256, + "input_cost": 0.036816, + "output_cost": 0.16644, + "prompt_tokens": 7984, + "completion_tokens": 1312 + } + }, + { + "name": "gpt-5.6-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ed318a18ec07 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788262, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer ed318a18ec07" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2d376f5f39a0 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788263, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 2d376f5f39a0" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12928, + "completion_tokens": 380, + "total_tokens": 13308, + "prompt_tokens_details": { + "cached_tokens": 12288 + } + } + } + }, + "expected": { + "spend": 0.0085904, + "input_cost": 0.0032704, + "output_cost": 0.00532, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "gpt-5.6-audio_input", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1eed63f65da0 summarize the attached material in one line and name the city weather" + }, + { + "type": "input_audio", + "input_audio": { + "data": "UklGRmQGAABXQVZFZm10IBAAAAABAAEAQB8AAIA+AAACABAAZGF0YUAGAAAAAA8I4A85F+EdpCNYKNkrCy7eLkwuWSwTKZUk/x59GEERgQl4AWb5hPER6kbjVd1t2LLUQdIu0X/RM9M81oTa6t9E5mPtD/UP/SQFEw2fFI0bqSHEJrgqZi27LqouNi1pKlkmJSH0GvUTXgxpBFP8WPS27KflYt8U2ujV/dJp0TjRbNL71NPY1d3b47nqOPIh+jUCOQrwER0ZjB8JJW0plCxoLtou5i2VK/cnKSNOHZUWLw9VB0T/OPdv7yToj+Hi20nX5tPT0SDR09Hm00nX4tuP4STob+8490T/VQcvD5UWTh0pI/cnlSvmLdouaC6ULG0pCSWMHx0Z8BE5CjUCIfo48rnq2+PV3dPY+9Rs0jjRadH90ujVFNpi36fltuxY9FP8aQReDPUT9BolIVkmaSo2Laouuy5mLbgqxCapIY0bnxQTDSQFD/0P9WPtRObq34TaPNYz03/RLtFB0rLUbdhV3UbjEeqE8Wb5eAGBCUERfRj/HpUkEylZLEwu3i4LLtkrWCikI+EdORfgDw8IAADx9yDwx+gf4lzcqNcn1PXRItG00afT7dZr2wHhg+e/7n/2iP6aBnwO7xW6HKsikydOK78t0i6BLs0sxCl8JRYgvBmdEvEK8QLc+u3yYetz5FfePNlI1ZrSRdFW0crSl9Wn2dveDOUL7KLzl/utA6gLShNZGp4g7CUYKgMtly7ILpQtBSstJysiJRxHFcgN3wXL/cf1EO7j5nTg99qT1mzTmNEm0RrSa9QJ2NfcsuJr6dHwq/i8AMgIkRDcF3EeHiS3KBosLS7gLi0uGiy3KB4kcR7cF5EQyAi8AKv40fBr6bLi19wJ2GvUGtIm0ZjRbNOT1vfadODj5hDux/XL/d8FyA1HFSUcKyItJwUrlC3ILpcuAy0YKuwlniBZGkoTqAutA5f7ovML7Azl296n2ZfVytJW0UXRmtJI1TzZV95z5GHr7fLc+vEC8QqdErwZFiB8JcQpzSyBLtIuvy1OK5MnqyK6HO8VfA6aBoj+f/a/7oPnAeFr2+3Wp9O00SLR9dEn1KjXXNwf4sfoIPDx9wAADwjgDzkX4R2kI1go2SsLLt4uTC5ZLBMplST/Hn0YQRGBCXgBZvmE8RHqRuNV3W3YstRB0i7Rf9Ez0zzWhNrq30TmY+0P9Q/9JAUTDZ8UjRupIcQmuCpmLbsuqi42LWkqWSYlIfQa9RNeDGkEU/xY9Lbsp+Vi3xTa6NX90mnRONFs0vvU09jV3dvjueo48iH6NQI5CvARHRmMHwklbSmULGgu2i7mLZUr9ycpI04dlRYvD1UHRP8492/vJOiP4eLbSdfm09PRINHT0ebTSdfi24/hJOhv7zj3RP9VBy8PlRZOHSkj9yeVK+Yt2i5oLpQsbSkJJYwfHRnwETkKNQIh+jjyuerb49Xd09j71GzSONFp0f3S6NUU2mLfp+W27Fj0U/xpBF4M9RP0GiUhWSZpKjYtqi67LmYtuCrEJqkhjRufFBMNJAUP/Q/1Y+1E5urfhNo81jPTf9Eu0UHSstRt2FXdRuMR6oTxZvl4AYEJQRF9GP8elSQTKVksTC7eLgsu2StYKKQj4R05F+APDwgAAPH3IPDH6B/iXNyo1yfU9dEi0bTRp9Pt1mvbAeGD57/uf/aI/poGfA7vFbocqyKTJ04rvy3SLoEuzSzEKXwlFiC8GZ0S8QrxAtz67fJh63PkV9482UjVmtJF0VbRytKX1afZ294M5QvsovOX+60DqAtKE1kaniDsJRgqAy2XLsgulC0FKy0nKyIlHEcVyA3fBcv9x/UQ7uPmdOD32pPWbNOY0SbRGtJr1AnY19yy4mvp0fCr+LwAyAiRENwXcR4eJLcoGiwtLuAuLS4aLLcoHiRxHtwXkRDICLwAq/jR8GvpsuLX3AnYa9Qa0ibRmNFs05PW99p04OPmEO7H9cv93wXIDUcVJRwrIi0nBSuULcguly4DLRgq7CWeIFkaShOoC60Dl/ui8wvsDOXb3qfZl9XK0lbRRdGa0kjVPNlX3nPkYevt8tz68QLxCp0SvBkWIHwlxCnNLIEu0i6/LU4rkyerIroc7xV8DpoGiP5/9r/ug+cB4Wvb7dan07TRItH10SfUqNdc3B/ix+gg8PH3", + "format": "wav" + } + } + ] + } + ], + "stream": false, + "modalities": [ + "text" + ], + "allowed_openai_params": [ + "modalities" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788262, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 1eed63f65da0" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1546, + "completion_tokens": 210, + "total_tokens": 1756, + "prompt_tokens_details": { + "audio_tokens": 1450 + } + } + } + }, + "expected": { + "spend": 0.061108, + "input_cost": 0.058168, + "output_cost": 0.00294, + "prompt_tokens": 1546, + "completion_tokens": 210 + } + }, + { + "name": "gpt-5.6-audio_output", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c2f69182025b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "modalities": [ + "text", + "audio" + ], + "audio": { + "voice": "alloy", + "format": "pcm16" + }, + "allowed_openai_params": [ + "modalities", + "audio" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788262, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer c2f69182025b" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 220, + "completion_tokens": 1300, + "total_tokens": 1520, + "completion_tokens_details": { + "audio_tokens": 1120 + } + } + } + }, + "expected": { + "spend": 0.092505, + "input_cost": 0.000385, + "output_cost": 0.09212, + "prompt_tokens": 220, + "completion_tokens": 1300 + } + }, + { + "name": "gpt-5.6-reasoning", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "839418b0b1da summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "reasoning_effort": "medium", + "allowed_openai_params": [ + "reasoning_effort" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788263, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 839418b0b1da" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1240, + "completion_tokens": 4040, + "total_tokens": 5280, + "completion_tokens_details": { + "reasoning_tokens": 3480 + } + } + } + }, + "expected": { + "spend": 0.06569, + "input_cost": 0.00217, + "output_cost": 0.06352, + "prompt_tokens": 1240, + "completion_tokens": 4040 + } + }, + { + "name": "gpt-5.6-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "fa273468c07b summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788262, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer fa273468c07b" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "flex" + } + }, + "expected": { + "spend": 0.004494, + "input_cost": 0.00161, + "output_cost": 0.002884, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "dbb27812caea summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788262, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer dbb27812caea" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + }, + "service_tier": "priority" + } + }, + "expected": { + "spend": 0.017976, + "input_cost": 0.00644, + "output_cost": 0.011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-web_search_medium", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2fc2074db6f0 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "medium" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788263, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 2fc2074db6f0", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + }, + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.021488, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-web_search_low", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8a1c27e0ad34 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "low" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788262, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 8a1c27e0ad34", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.018988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-web_search_high", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "14ebe654d39f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "web_search_options": { + "search_context_size": "high" + }, + "allowed_openai_params": [ + "web_search_options" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788263, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 14ebe654d39f", + "annotations": [ + { + "type": "url_citation", + "url_citation": { + "url": "https://scripted.example/source", + "title": "scripted source", + "start_index": 0, + "end_index": 1 + } + } + ] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.023988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0d4dc45197bd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788262, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788262, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 0d4dc45197bd\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788262, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788262, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d2437c6d35d6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer d2437c6d35d6\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05 + } + } + }, + { + "name": "gpt-5.6-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "510682506548 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05 + } + } + }, + { + "name": "gpt-5.6-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "93ef594b4d91 summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 93ef594b4d91\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.75e-06, + "output_cost_per_token": 1.4e-05 + } + } + }, + { + "name": "gpt-5.6-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e6d045b77d68 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788263, + "model": "gpt-5.4-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer e6d045b77d68" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "49c74a898360 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 49c74a898360\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.4-mini\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.4-mini\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0017976, + "input_cost": 0.000644, + "output_cost": 0.0011536, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8a209834c60c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788263, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "77c2cb29e969 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788264, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.008988, + "input_cost": 0.00322, + "output_cost": 0.005768, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "gpt-5.6-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "gpt-5.6", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "66e5a1e22691 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 66e5a1e22691\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788263, \"model\": \"gpt-5.6\", \"choices\": [], \"usage\": {\"prompt_tokens\": 8314, \"completion_tokens\": 1592, \"total_tokens\": 9906, \"prompt_tokens_details\": {\"cached_tokens\": 6144, \"audio_tokens\": 330}, \"completion_tokens_details\": {\"reasoning_tokens\": 900, \"audio_tokens\": 280}}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0600632, + "input_cost": 0.0174952, + "output_cost": 0.042568, + "prompt_tokens": 8314, + "completion_tokens": 1592 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "54c4ce4d8096 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 54c4ce4d8096" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_read_at_input_rate", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "36b591711f22 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 36b591711f22" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 640, + "outputTokens": 380, + "totalTokens": 13308, + "cacheReadInputTokens": 12288 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.00347132, + "input_cost": 0.00310272, + "output_cost": 0.0003686, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-fallback_cache_write_at_input_rate", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3bd1faf7cebd summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 3bd1faf7cebd" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 512, + "outputTokens": 350, + "totalTokens": 10078, + "cacheWriteInputTokens": 9216, + "cacheDetails": [ + { + "inputTokens": 9216, + "ttl": "5m" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.00267422, + "input_cost": 0.00233472, + "output_cost": 0.0003395, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f515db6db1e8 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer f515db6db1e8" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c423409dd543 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer c423409dd543" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07 + } + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1b82e406f204 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockStart", + "payload": { + "start": { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler fil" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ler filler filler filler filler filler filler filler filler filler filler filler filler fi" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ller filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "tool_use" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07 + } + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "52555527573a summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 52555527573a" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 2.4e-07, + "output_cost_per_token": 9.7e-07 + } + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c47a40f71743 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer c47a40f71743" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1aa422adaa97 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 1aa422adaa97" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2e3593b273a4 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather", + "input": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3f3df4cdd7d9 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockStart", + "payload": { + "start": { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler fil" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ler filler filler filler filler filler filler filler filler filler filler filler filler fi" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ller filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "tool_use" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.00084124, + "input_cost": 0.0004416, + "output_cost": 0.00039964, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "meta.llama4-maverick-17b-instruct-v1:0-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "meta.llama4-maverick-17b-instruct-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5eec826ded90 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 5eec826ded90" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 11468, + "cacheReadInputTokens": 6144, + "cacheWriteInputTokens": 3072, + "cacheDetails": [ + { + "inputTokens": 2048, + "ttl": "5m" + }, + { + "inputTokens": 1024, + "ttl": "1h" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.00305308, + "input_cost": 0.00265344, + "output_cost": 0.00039964, + "prompt_tokens": 11056, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d6c7504381ab summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788266, + "model": "moonshotai/Kimi-K3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer d6c7504381ab" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1fb11cd276fc summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788266, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788266, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 1fb11cd276fc\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788266, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788266, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "316d5b71455c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 316d5b71455c\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.15e-06, + "output_cost_per_token": 3.45e-06 + } + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "d522e5409f42 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.15e-06, + "output_cost_per_token": 3.45e-06 + } + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "97f79b9004cf summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 97f79b9004cf\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 1.15e-06, + "output_cost_per_token": 3.45e-06 + } + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "10e55a5c4a81 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788268, + "model": "zai-org/GLM-5.3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 10e55a5c4a81" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "598ed6ff4b9d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 598ed6ff4b9d\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788269, \"model\": \"zai-org/GLM-5.3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "761da386a9ae summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788269, + "model": "moonshotai/Kimi-K3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e33dced1d70c summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-moonshotai-Kimi-K3-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/moonshotai/Kimi-K3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "2e2f3465f331 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 2e2f3465f331\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788268, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5438abd6c548 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788271, + "model": "zai-org/GLM-5.3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 5438abd6c548" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "25be31c2d005 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 25be31c2d005\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "3a906f4aa16d summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 3a906f4aa16d\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.2e-06 + } + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1143ec257764 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.2e-06 + } + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "79e942a4452d summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 79e942a4452d\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: [DONE]" + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.5e-07, + "output_cost_per_token": 2.2e-06 + } + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "29dffdfbd5fa summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788271, + "model": "moonshotai/Kimi-K3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "scripted answer 29dffdfbd5fa" + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "5985ca98af28 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 5985ca98af28\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788270, \"model\": \"moonshotai/Kimi-K3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0035374, + "input_cost": 0.002116, + "output_cost": 0.0014214, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "60392e73043e summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "application/json", + "body": { + "id": "chatcmpl-$REQUEST_ID", + "object": "chat.completion", + "created": 1789788270, + "model": "zai-org/GLM-5.3", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_$REQUEST_ID", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 1840, + "completion_tokens": 412, + "total_tokens": 2252 + } + } + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f2bebf9a77ac summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "tool_choice": "auto", + "allowed_openai_params": [ + "tool_choice" + ] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"tool_calls\": [{\"index\": 0, \"id\": \"call_$REQUEST_ID\", \"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"arguments\": \"\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"{\\\"city\\\": \\\"Berlin\\\", \\\"days\\\": 7, \\\"units\\\": \\\"metric\\\", \\\"notes\\\": \\\"filler filler filler filler fil\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ler filler filler filler filler filler filler filler filler filler filler filler filler fi\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"tool_calls\": [{\"index\": 0, \"function\": {\"arguments\": \"ller filler filler filler filler filler filler filler filler filler filler filler filler \\\"}\"}}]}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"tool_calls\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788271, \"model\": \"zai-org/GLM-5.3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "together_ai-zai-org-GLM-5.3-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "together_ai/zai-org/GLM-5.3", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "63a7c8ddf892 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "text/event-stream", + "frames": [ + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788272, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788272, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {\"role\": \"assistant\", \"content\": \"scripted answer 63a7c8ddf892\"}, \"finish_reason\": null}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788272, \"model\": \"zai-org/GLM-5.3\", \"choices\": [{\"index\": 0, \"delta\": {}, \"finish_reason\": \"stop\"}], \"usage\": null}", + "data: {\"id\": \"chatcmpl-$REQUEST_ID\", \"object\": \"chat.completion.chunk\", \"created\": 1789788272, \"model\": \"zai-org/GLM-5.3\", \"choices\": [], \"usage\": {\"prompt_tokens\": 1840, \"completion_tokens\": 412, \"total_tokens\": 2252}}", + "data: [DONE]" + ] + }, + "expected": { + "spend": 0.0019184, + "input_cost": 0.001012, + "output_cost": 0.0009064, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-input_text", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "f6559891a89a summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer f6559891a89a" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-cache_read", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "c054e1cd6b20 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer c054e1cd6b20" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 640, + "outputTokens": 380, + "totalTokens": 13308, + "cacheReadInputTokens": 12288 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.0207284, + "input_cost": 0.0102784, + "output_cost": 0.01045, + "prompt_tokens": 12928, + "completion_tokens": 380 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-cache_write_5m", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "87e62170eee7 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 87e62170eee7" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 512, + "outputTokens": 350, + "totalTokens": 10078, + "cacheWriteInputTokens": 9216, + "cacheDetails": [ + { + "inputTokens": 9216, + "ttl": "5m" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.075801, + "input_cost": 0.066176, + "output_cost": 0.009625, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-cache_write_1h", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4566b7a4b0d6 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 4566b7a4b0d6" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 512, + "outputTokens": 350, + "totalTokens": 10078, + "cacheWriteInputTokens": 9216, + "cacheDetails": [ + { + "inputTokens": 2048, + "ttl": "5m" + }, + { + "inputTokens": 7168, + "ttl": "1h" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.105369, + "input_cost": 0.095744, + "output_cost": 0.009625, + "prompt_tokens": 9728, + "completion_tokens": 350 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-tiered_input_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e6889b23c228 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer e6889b23c228" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 204800, + "outputTokens": 620, + "totalTokens": 205420 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 2.278375, + "input_cost": 2.2528, + "output_cost": 0.025575, + "prompt_tokens": 204800, + "completion_tokens": 620 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-tiered_cache_read_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "1d35f19047ff summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 1d35f19047ff" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 4096, + "outputTokens": 480, + "totalTokens": 206304, + "cacheReadInputTokens": 201728 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.2867568, + "input_cost": 0.2669568, + "output_cost": 0.0198, + "prompt_tokens": 205824, + "completion_tokens": 480 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-tiered_cache_write_above_200k", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "14f144dc9bee summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 14f144dc9bee" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 4096, + "outputTokens": 480, + "totalTokens": 205280, + "cacheWriteInputTokens": 200704, + "cacheDetails": [ + { + "inputTokens": 200704, + "ttl": "5m" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 2.824536, + "input_cost": 2.804736, + "output_cost": 0.0198, + "prompt_tokens": 204800, + "completion_tokens": 480 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-service_tier_flex", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e984661f7bde summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "flex", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer e984661f7bde" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + }, + "serviceTier": { + "type": "flex" + } + } + }, + "expected": { + "spend": 0.010725, + "input_cost": 0.00506, + "output_cost": 0.005665, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-service_tier_priority", + "covers": "quota_management.spend_tracking.cost_matrix.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "419bc91d93ae summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "service_tier": "priority", + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 419bc91d93ae" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + }, + "serviceTier": { + "type": "priority" + } + } + }, + "expected": { + "spend": 0.0268125, + "input_cost": 0.01265, + "output_cost": 0.0141625, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "4a3e5a729480 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 4a3e5a729480" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream_no_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "45449a962a21 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 45449a962a21" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.5e-06, + "output_cost_per_token": 2.75e-05 + } + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream_no_usage_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "e0e1b17ca05f summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockStart", + "payload": { + "start": { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler fil" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ler filler filler filler filler filler filler filler filler filler filler filler filler fi" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ller filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "tool_use" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.5e-06, + "output_cost_per_token": 2.75e-05 + } + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream_no_usage_image_input", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7d1ebbfd135c summarize the attached material in one line and name the city weather" + }, + { + "type": "image_url", + "image_url": { + "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAMK0lEQVR4nAEgDN/zAM0HLNi+b59irEwJwoIG5+NVlKprNC9dCjpeSEL6tCj3YubiguXBZXx4w6lns2cR6zkGp8hgPXHUCeelTYe9wfcEQgJ6rx+pW3+GWJV430PkExZ66Nnc6zd2KDOBGnGnIwBzhiZIL2HGI3lifMEk1EYYPG5NnqGlpcz3LiFAwwS9/Goh5egSF1aINdSX+yErhrSeZWrPBkEWmgtZ9OYpQ58l2dRlT+yNSBn7QNa6ssjgEhxEGuYUp7jZKpIZrx7Oh1QAV1jeeB7zT4/0jccZhxGSWqLiJW9VRPJQuRZjnL/J8qO8F7vpSKdYNMGDc/cEMXKNBlAdehNaVHGe84Tdmm53hcOfr0ICjvEPuk0WzuaDIOumjneMSZ1+6qg8mAM8quAXAOyQPrjRNxDXsUwZZhYr07VMCinT0OP4yIEWDKvlaxGgReVKAJ1JpZx/Hlt+evT70yo3G94iWEhVavFwPnqJ87rKl0BT6+ohtOgz197MvB8QzMXpME/BwepPYkiRLpbBOAD37hU9bAWozdK3sPczg3okfSudzcFjAYtEIq5yw+1ZF9EYmBSexEP+IhnvUWC4BeDWZQiCjhF7/4syzu4C5kF9ETfrG+udK0232R+NA+uESn014bRJmPMfahYljDojL1UAbuaA0PuOGOzBBlCKXAkFNHIfvvZHQafMkl9qmqdFF4x3Em6WDuijSQXN6nFtM3UXbUGmmCF4Rcyl4YhifPspUXLdXZMIvPo9zAhTSlQFEi8m83sw5KxL0rWBzS9c4XAIAOaz3py4dTb7d9QaqDMLk0J87/15PZKvEaC6/hZ62sCtuFTywatjViHrBXTgOO5IJsiyYuzmaeQJKHmr11gnixR2rO7lqNEGs3shT+xK/lDUucFkigvA+a5C+itk/VV+1gDxc420UHpKhj31j0YnCZSFJebGz2/XSTyT6XfZhG4XNwZGIeUGCPKt0DX9lpB0RNN0yiPzQVJfa3LkZpRBN3RGgRpYc8WpHn5Q1wWpmnklpPHACv/N+kGz0Ki86g2GgvsAWloXy5pwfFtwZRYV9fMGU4Za35yfj4cddJuHfAyHSpYHVlGhNknUVQIBV9gOq7wwLZU3Pl5GJgSy4EK7f75iRVKD/B0ItpC0FBpwODhWP184ymnLgLGkK90WIVUY7hZtAEOu39BF2OsPDWrBGWN0esj6v3clX2z22gaLmrJHiwE4sXWUC8TLLtFp4uiS/VlbojLP9uiewb/vrTLBiFjYJ5rsFjuu/3XxEuiZ1QYyi9sfsFqPoiXjQjCA/jibX4aA1AAfp3GT1lykHtVMJmSxoW4Xvn3BXhXUdtWhIwP7NDO1HRP9UAlEUgKbd/iJBWS20DFBJQb2/UN/+CpSWi97Rta3+pe3H4kOr3qaV+g1Udgmupu6/cxmQqMP+DXd77Sy6a0AMxTVBR0DU4sVW/Vs2qPfnh3r+xlqt/3VIhyKQknN6xF5RIg4j7xsEpjsnKV/XhJNmN2sWekxom9lUCkuP3qgD25S7oCG6ZV3CrkUCm48s5hw+dUZANcGs4H6/PxIrSpkAC77CDPFF5hCvkfKWzerhufLBkq7AhZgeMKRnNboaP3m9qMh6+9Z3JEyaV8rfUacshIsMqwS/BI0uL9v9+fwcMQrbdwOstrkyWCPG61dXIAoEb9t2HnSdSlxy6FXa5+LhwCvCy1ANKkBHgVSx5hoE+LrBX47caskZapZ+MAsTFJhA3VxvHgKaWiuX4/vaG7TbOZdX7GRSzPz3yOeM4JeAuLqwOy6T0OBIKbnSm5b3A5+Y2j2cNYMiVmogx89QCIPRicA936Dj6rC2bDKBi8DmXw8dZvR170ELj4UjqD+VRopML3ww7ILyoVYi5P150ekt4QioS95PZdaHcPXSAD0GwVZe1t0K1qy2TGc7V2ySUwrZKwEnPRbLXAcl6praPJ8aVbkAJ1MPaI09ZDaZOT+npRQ4SFv1DK3halvT/cYVWNFxZy/HEwXatv4MtSF+5ymHEWqFC/kYwCJMTaYtzI7Ma9Q1rJ1WZtVBP36KFFdSj1G8Bw5b5ssozn/uHJxFO9g937ZtQBQvxvgA4h8rApfcpEHs+HfrYkWalhdEwiJ+fpmF/Um3ywbq7MF3kWROuUQa4H2rcVhq4WpdGuBte/A+QvHrGkq45kCcmvaWhAAslxCV5CWs7QlXig29URyTAgPh//Ji+IAJXC9fNNMdeirO7iPEEOemj9zZ8FMiAQAYKRF4o8E9glP+Jx+Vw9xU5oM40/363XW5T+Gd4xtwwwQoR3P38Wd0uURAeDvp2P53Wz6zxpyRmpc2iAwN5BujDYD2v+lpIn2AMUaEqKkrYP6sRheFZ0EY9Yt7r25gt9pITZVoPxxTMUDlHfdZgyMFfPLKLOtUk3gaiv88FD1Wd4AdI2pNnmYqAMKjqK35Ys3wVuBmgAbUPH6k4ae0ku7/qyRrkGHwRewnAAVZQgZavvBMFJ8cB4Ndb2bXEI2phv40TLtxqffTyNrTfbyrEdKKESwv/h/+pkLpi4bcaUZx0fBeRewm9oUozrsfuGNZcn0rKC8DdMUtqqXBWClU0bAHPTp2GD2c1FNwcwAFOXWyuK3ox0nFYLckD+pQW3ZJyKBKlCnanFXhVDRONHgupmm89QapXZ32FhAzncd/Xcy34ljiSee3UVHJtnq2YJJwQQVBNQKjuhoCv0YqzT/zVWuTbF5QtRm8I7cC5FQANj9TfJqwOxdaoY98K6RRZGzAezoRsF02Q3PwAowC6vYqUzHv85kEfb7BYqzoXGLm4fLxafznJZ+JhJdtsmV5qQiAxjnFQdrdTtL4KNZCXlvtdVYXxLGFN9os7WJoU2kLQB5RTU1rvAlYHfbTZZTTYFKXxRHAizqcSNldKkms7p4RYVAWlUXIGdWTb4kym3qAF4clO+3Njf9F3zxl1b73YfH8pXbnk9fIQnHRowK9vQFRcfD8ikeIUAmZrhe+3cOXZUAt7EeSjZgZFxWFhFll/ZC/Y93aYwMJjAhvbgcS/zZaR0ucmJ663gDu7xgXT3tgQrvb4daDFM10U5HZ/kt2r9m26me5oXdgom65hEkEgfTe03ZFzZnPz7n1fzuGVThCppMAAAyK2pIlNAb40xyGcWNko8VyJ3oghsue2leWHkJ6UpWCdZB13x+QcxkLOr5a8X0zx6N+VcXwNMfGGqlelK0sh1MrRizk4drf0prMW3Q264J3IVJTZ3tx1ULpBi8CnnmrwBeYvfkI5rZJ2W6cOv3bKMqoCpyoNo+gpCZcSVgB8hdzlfMfPl67dL9/IqNo2mBQBaVitoQ4y7KzqHnrBWSxLIyznP3v/VNlzgF7qcPaQiDYhyGAgLfihsZE5nQZLcedY4Apm3dhEOyMUpdzNNyViCP6ykfFn35Ss3JDUS7lfZV47bpxeatoe2dNzU5TGxY9GoVYvhKXmP0WjgOpwv91XPF/x1ty0JRgNGVMwnwxk4PpSpDgGRu/k4jMev9x1vlc3EdAIz+WBCWU23SZoIT3WfyVNpXi7azEQkSB7e1K+fS25uGiNScc4/Nn8BVsUqtwtGhM5nncB7tH0F/k0ngZeGmqEGNJw01v2UYpLQoxwqhCz7/X5it8liVJMxk2kQ3iCGYnwDwfHcl7gLJGiAqrjLBOlp17bPGYIGTlFw+TOCLsGS103TZvSbyptAazsca/TY3o5hZLqX7SBvnfL6s/SU7BVyQY4PpZwHLa6PI2w+uFmJZbBsaruCVPW+FL+zSfvPO560ABJLD6nKsReNBwoxShnNi1XPSmX+C7Ij/ik2nfzubk6IFlomrI0bs386ztTMDifWKNO0ZnC+Oau6ZgwE4PwF8AqKtGFd5yqG0aMFq9rvwPb8DZCXmUHN5Uc40cNG5HWNSAK1yFAkCcG2GPV/7JYEi3t9ZA8NBeh9Dmvsjv8jiaeknPCij14x6BmGOFzyVNqRcS8x517fM39S0cC6bzu8wbnh96fwQ9z/JzCEsqxV9KjuES+xv3rYkX+r3JxcK7p78EACPip85gRBoIU7cZgrwLepMArrpA9YvR8Icag3YILzWcgXVoVRTs6Xc2PsiKasZ98y5CBkmP9Ri3lwZ9acgvPJ1fw729yDzDl/0qEeBOf7ZYZ7dreaPeaGHIymfgKAwmwhnjvAGaqLWhgAAAABJRU5ErkJggg==", + "detail": "high" + } + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 7d1ebbfd135c" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + } + ] + }, + "expected": { + "recount": { + "input_cost_per_token": 5.5e-06, + "output_cost_per_token": 2.75e-05 + } + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "80542567b1bb summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "text": "scripted answer 80542567b1bb" + } + ] + } + }, + "stopReason": "end_turn", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream_response_model_override", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "ab06cda24199 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer ab06cda24199" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "7a6c5d71a8fe summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": false, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/json", + "body": { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather", + "input": { + "city": "Berlin", + "days": 7, + "units": "metric", + "notes": "filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler filler " + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream_tool_call", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line." + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "0ef1034f8717 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather and a short forecast for a city.", + "parameters": { + "type": "object", + "properties": { + "city": { + "type": "string", + "description": "City name" + }, + "days": { + "type": "integer", + "description": "Forecast horizon in days" + }, + "units": { + "type": "string", + "enum": [ + "metric", + "imperial" + ] + } + }, + "required": [ + "city" + ] + } + } + } + ], + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockStart", + "payload": { + "start": { + "toolUse": { + "toolUseId": "tooluse_$REQUEST_ID", + "name": "get_weather" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "{\"city\": \"Berlin\", \"days\": 7, \"units\": \"metric\", \"notes\": \"filler filler filler filler fil" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ler filler filler filler filler filler filler filler filler filler filler filler filler fi" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "toolUse": { + "input": "ller filler filler filler filler filler filler filler filler filler filler filler filler \"}" + } + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "tool_use" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 2252 + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.02145, + "input_cost": 0.01012, + "output_cost": 0.01133, + "prompt_tokens": 1840, + "completion_tokens": 412 + } + }, + { + "name": "us.anthropic.claude-opus-5-v1:0-stream_full_usage", + "covers": "quota_management.spend_tracking.scripted_wire.logs_cost", + "model": "us.anthropic.claude-opus-5-v1:0", + "request": { + "model": "$MODEL", + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", + "cache_control": { + "type": "ephemeral", + "ttl": "1h" + } + } + ] + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "8ba9788e0bd5 summarize the attached material in one line and name the city weather" + } + ] + } + ], + "stream": true, + "stream_options": { + "include_usage": true + }, + "allowed_openai_params": [] + }, + "response": { + "content_type": "application/vnd.amazon.eventstream", + "events": [ + { + "event_type": "messageStart", + "payload": { + "role": "assistant" + } + }, + { + "event_type": "contentBlockDelta", + "payload": { + "delta": { + "text": "scripted answer 8ba9788e0bd5" + }, + "contentBlockIndex": 0 + } + }, + { + "event_type": "contentBlockStop", + "payload": { + "contentBlockIndex": 0 + } + }, + { + "event_type": "messageStop", + "payload": { + "stopReason": "end_turn" + } + }, + { + "event_type": "metadata", + "payload": { + "usage": { + "inputTokens": 1840, + "outputTokens": 412, + "totalTokens": 11468, + "cacheReadInputTokens": 6144, + "cacheWriteInputTokens": 3072, + "cacheDetails": [ + { + "inputTokens": 2048, + "ttl": "5m" + }, + { + "inputTokens": 1024, + "ttl": "1h" + } + ] + }, + "metrics": { + "latencyMs": 42 + } + } + } + ] + }, + "expected": { + "spend": 0.0501732, + "input_cost": 0.0388432, + "output_cost": 0.01133, + "prompt_tokens": 11056, + "completion_tokens": 412 + } + } + ] +} diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py new file mode 100644 index 00000000000..a8a56fbfbbd --- /dev/null +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -0,0 +1,101 @@ +"""Cost tracking coverage for literal integration request and response data.""" + +from __future__ import annotations + +from hashlib import sha256 +from typing import Final, cast + +import pytest + +from integration._support.client import JSON_OBJECT, Gateway +from integration.cost_calculation.conftest import ( + approx_equal, + assert_total_is_sum_of_components, + poll_cost_row, + register_scenario_deployment, +) +from integration.cost_calculation.cost_tracking_case import ( + CASES, + CostTrackingTestCase, + ExactExpected, + RecountExpected, + data_errors, +) + +if _data_errors := data_errors(): + raise ValueError("\n".join(_data_errors)) + + +_CASES: Final = tuple( + pytest.param(case, marks=pytest.mark.covers(case.covers), id=case.name) + for case in CASES +) + + +def _assert_stream_has_no_error(response_text: str) -> None: + for line in response_text.splitlines(): + if not line.startswith("data:"): + continue + payload = line.removeprefix("data:").strip() + if payload == "[DONE]": + continue + parsed = JSON_OBJECT.validate_json(payload) + assert "error" not in parsed, f"stream carried an error event: {parsed}" + + +@pytest.mark.parametrize("case", _CASES) +def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) -> None: + marker: Final = sha256(case.name.encode()).hexdigest()[:12] + with gateway.scenario() as scenario: + key: Final = scenario.key() + model_name: Final = register_scenario_deployment(scenario, case, marker, key) + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + {**case.request, "model": model_name}, + key=key, + ) + assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" + if case.response.content_type == "text/event-stream": + _assert_stream_has_no_error(response.text) + row: Final = poll_cost_row(key) + if isinstance(case.expected, RecountExpected): + assert row.prompt_tokens is not None and row.prompt_tokens > 0, ( + f"{case.name}: recount case counted no input tokens: prompt_tokens={row.prompt_tokens}" + ) + assert row.completion_tokens is not None and row.completion_tokens > 0, ( + f"{case.name}: recount case counted no output tokens: completion_tokens={row.completion_tokens}" + ) + recount: Final = row.prompt_tokens * case.expected.recount.input_cost_per_token + ( + row.completion_tokens * case.expected.recount.output_cost_per_token + ) + assert row.spend is not None and approx_equal(row.spend, recount), ( + f"{case.name}: spend {row.spend} != recount {recount} at map rates" + ) + assert_total_is_sum_of_components(row, case.name) + return + expected: Final = case.expected + assert isinstance(expected, ExactExpected) + if case.response.content_type == "application/json": + header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) + assert header is not None and approx_equal(float(header), expected.spend), ( + f"{case.name}: x-litellm-response-cost {header} != expected {expected.spend}" + ) + assert row.spend is not None and approx_equal(row.spend, expected.spend), ( + f"{case.name}: spend {row.spend} != expected {expected.spend} " + f"(breakdown {row.breakdown.model_dump()})" + ) + breakdown: Final = row.breakdown + assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, expected.input_cost), ( + f"{case.name}: input_cost {breakdown.input_cost} != expected {expected.input_cost}" + ) + assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, expected.output_cost), ( + f"{case.name}: output_cost {breakdown.output_cost} != expected {expected.output_cost}" + ) + assert row.prompt_tokens == expected.prompt_tokens, ( + f"{case.name}: prompt_tokens {row.prompt_tokens} != expected {expected.prompt_tokens}" + ) + assert row.completion_tokens == expected.completion_tokens, ( + f"{case.name}: completion_tokens {row.completion_tokens} != expected {expected.completion_tokens}" + ) + assert_total_is_sum_of_components(row, case.name) diff --git a/tests/integration/cost_calculation/test_token_pricing.py b/tests/integration/cost_calculation/test_token_pricing.py deleted file mode 100644 index cc48da2b819..00000000000 --- a/tests/integration/cost_calculation/test_token_pricing.py +++ /dev/null @@ -1,245 +0,0 @@ -"""Token pricing coverage for the integration scripted-shape cost shard.""" - -from __future__ import annotations - -import uuid -from typing import Final, cast - -import pytest -from pydantic import JsonValue - -from integration._support.client import JSON_OBJECT, Gateway -from integration._support.scripted_shapes import ScriptedUsage, Shape -from integration.cost_calculation.conftest import ( - approx_equal, - assert_total_is_sum_of_components, - poll_cost_row, - register_scenario_deployment, -) -from integration.cost_calculation.cost_matrix import ( - AUDIO_INPUT_DATA_URL, - FRONTIER_MODELS, - IMAGE_INPUT_DATA_URL, - SERVICE_TIER_REQUEST_SHAPES, - VIDEO_INPUT_DATA_URL, - Case, - FrontierModel, - cases_for, - matrix_data_errors, - recount_cost, -) - -if _data_errors := matrix_data_errors(): - raise ValueError("\n".join(_data_errors)) - -def _case_id(param: tuple[FrontierModel, Case]) -> str: - model, case = param - return f"{model.map_key.replace('/', '-')}-{case.name}" - - -_MATRIX: Final = tuple( - pytest.param( - (model, case), - marks=pytest.mark.covers( - "quota_management.spend_tracking.scripted_wire.logs_cost" - if case.family == "transport" - else "quota_management.spend_tracking.cost_matrix.logs_cost" - ), - id=_case_id((model, case)), - ) - for model in FRONTIER_MODELS - for case in cases_for(model) -) -_CACHE_SHAPES: Final = frozenset({"anthropic_messages", "bedrock_converse"}) -_WEB_SEARCH_OPTION_SHAPES: Final = frozenset({"openai_chat", "openai_responses"}) - - -def _cache_control(usage: ScriptedUsage, shape: Shape) -> dict[str, JsonValue] | None: - if shape not in _CACHE_SHAPES: - return None - if not (usage.cache_read_tokens or usage.cache_write_5m_tokens or usage.cache_write_1h_tokens): - return None - return {"type": "ephemeral", **({"ttl": "1h"} if usage.cache_write_1h_tokens else {})} - - -def _chat_body(model: FrontierModel, case: Case, model_name: str, marker: str) -> dict[str, JsonValue]: - usage: Final = case.usage_for(model.map_key) - user_parts: Final = [ - {"type": "text", "text": f"{marker} summarize the attached material in one line and name the city weather"}, - *( - [{"type": "image_url", "image_url": {"url": IMAGE_INPUT_DATA_URL, "detail": "high"}}] - if case.image_input - else [] - ), - *( - [{"type": "input_audio", "input_audio": {"data": AUDIO_INPUT_DATA_URL.split(",", 1)[1], "format": "wav"}}] - if case.audio_input - else [] - ), - *( - [{"type": "file", "file": {"file_data": VIDEO_INPUT_DATA_URL, "format": "mp4"}}] - if case.video_input - else [] - ), - ] - tools: Final[list[JsonValue]] = [ - *( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the current weather and a short forecast for a city.", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"}, - "days": {"type": "integer", "description": "Forecast horizon in days"}, - "units": {"type": "string", "enum": ["metric", "imperial"]}, - }, - "required": ["city"], - }, - }, - } - ] - if case.tool_call - else [] - ), - *( - [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}] - if case.web_search is not None and model.shape == "anthropic_messages" - else [] - ), - *( - [{"googleSearch": {}}] - if case.web_search is not None and model.shape == "gemini_generate" - else [] - ), - *([{"googleMaps": {}}] if case.google_maps else []), - *([{"type": "file_search", "vector_store_ids": ["vs_cost_calc_fixture"]}] if case.file_search else []), - ] - cache_control: Final = _cache_control(usage, model.shape) - message: Final = { - "role": "system", - "content": [ - { - "type": "text", - "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line.", - **({"cache_control": cache_control} if cache_control else {}), - } - ], - } - return cast(dict[str, JsonValue], { - "model": model_name, - "messages": [message, {"role": "user", "content": user_parts}], - "stream": case.stream, - **({"stream_options": {"include_usage": True}} if case.stream else {}), - **( - {"service_tier": case.service_tier} - if case.service_tier is not None and model.shape in SERVICE_TIER_REQUEST_SHAPES - else {} - ), - **({"reasoning_effort": "medium"} if case.reasoning else {}), - **( - {"modalities": ["text", "audio"] if case.audio_output else ["text"]} - if case.audio_input or case.audio_output - else {} - ), - **({"audio": {"voice": "alloy", "format": "pcm16"}} if case.audio_output else {}), - **( - {"web_search_options": {"search_context_size": case.web_search}} - if case.web_search is not None and model.shape in _WEB_SEARCH_OPTION_SHAPES - else {} - ), - **({"tools": tools} if tools else {}), - **({"tool_choice": "auto"} if case.tool_call and model.shape != "bedrock_converse" else {}), - "allowed_openai_params": [ - name - for name, sent in ( - ("tool_choice", case.tool_call and model.shape != "bedrock_converse"), - ("modalities", case.audio_input or case.audio_output), - ("audio", case.audio_output), - ("web_search_options", case.web_search is not None), - ("reasoning_effort", case.reasoning), - ) - if sent - ], - }) - - -def _assert_stream_has_no_error(response_text: str) -> None: - for line in response_text.splitlines(): - if not line.startswith("data:"): - continue - payload = line.removeprefix("data:").strip() - if payload == "[DONE]": - continue - parsed = JSON_OBJECT.validate_json(payload) - assert "error" not in parsed, f"stream carried an error event: {parsed}" - - -@pytest.mark.parametrize("model_case", _MATRIX) -def test_scripted_usage_bills_at_map_rates( - gateway: Gateway, - model_case: tuple[FrontierModel, Case], -) -> None: - model, case = model_case - marker: Final = uuid.uuid4().hex[:12] - with gateway.scenario() as scenario: - key: Final = scenario.key() - model_name: Final = register_scenario_deployment(scenario, model, case, marker) - response: Final = gateway.request( - "POST", - "/v1/chat/completions", - _chat_body(model, case, model_name, marker), - key=key, - ) - assert response.is_success, ( - f"{model.map_key}/{case.name}: proxy returned {response.status_code}: {response.text[:400]}" - ) - if case.stream: - _assert_stream_has_no_error(response.text) - row: Final = poll_cost_row(key) - context: Final = f"{model.map_key}/{case.name}" - if not case.exact_spend: - assert row.prompt_tokens is not None and row.prompt_tokens > 0, ( - f"{context}: no-usage stream counted no input tokens: prompt_tokens={row.prompt_tokens}" - ) - assert row.completion_tokens is not None and row.completion_tokens > 0, ( - f"{context}: no-usage stream counted no output tokens: completion_tokens={row.completion_tokens}" - ) - if case.image_input: - assert row.prompt_tokens < 4000, ( - f"{context}: image data URL looks tokenized as text: prompt_tokens={row.prompt_tokens}" - ) - recount: Final = recount_cost(model, case, row.prompt_tokens, row.completion_tokens) - assert row.spend is not None and approx_equal( - row.spend, recount - ), f"{context}: no-usage stream spend {row.spend} != recount {recount} at map rates" - assert_total_is_sum_of_components(row, context) - return - golden: Final = case.expected_for(model) - if not case.stream: - header: Final = cast(str | None, response.headers.get("x-litellm-response-cost")) - assert header is not None and approx_equal(float(header), golden.spend), ( - f"{context}: x-litellm-response-cost {header} != golden {golden.spend}" - ) - assert row.spend is not None and approx_equal(row.spend, golden.spend), ( - f"{context}: spend {row.spend} != golden {golden.spend} " - f"(breakdown {row.breakdown.model_dump()})" - ) - breakdown: Final = row.breakdown - assert breakdown.input_cost is not None and approx_equal(breakdown.input_cost, golden.input_cost), ( - f"{context}: gross input_cost {breakdown.input_cost} != golden {golden.input_cost}; " - "cached/written tokens billed at the input rate" - ) - assert breakdown.output_cost is not None and approx_equal(breakdown.output_cost, golden.output_cost), ( - f"{context}: output_cost {breakdown.output_cost} != golden {golden.output_cost}" - ) - assert row.prompt_tokens == golden.prompt_tokens, ( - f"{context}: prompt_tokens {row.prompt_tokens} != golden {golden.prompt_tokens}" - ) - assert row.completion_tokens == golden.completion_tokens, ( - f"{context}: completion_tokens {row.completion_tokens} != golden {golden.completion_tokens}" - ) - assert_total_is_sum_of_components(row, context)