From bcb6a6eaab2f5080fb905014fc60f413df84fad3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 01:50:01 -0700 Subject: [PATCH 1/4] test(e2e): pin prompt-cache, service-tier, and cost-header billing Seven live e2e tests covering cost-tracking regressions that currently ship unnoticed: cache-write tokens billed at the cache-creation rate (#34046), per-component cost_breakdown on the spend row (#31686), cache reads billed at the cache-read discount on streamed calls (#34812), cache tokens surviving the anthropic-messages to Responses bridge (#34957), priority-tier rates applied to input, output and reasoning (#35923, #35925), the per-component response cost headers summing to the total (#36965), and cost injected into the final usage frame of an /openai passthrough stream (#36503). Every test registers its own deployment with a distinct custom rate per component, so a component billed at the wrong rate cannot pass. The shared helpers in cost_rows.py encode the one thing the two surfaces disagree on: the spend row's input_cost is gross of cache while the response's cost-input header is net of it. --- .../coverage_registry/quota_management.yaml | 7 + tests/e2e/models.py | 25 +- .../spend_tracking/cost_rows.py | 204 ++++++++++++++ .../test_cache_cost_accounting_e2e.py | 263 ++++++++++++++++++ .../spend_tracking/test_cost_headers_e2e.py | 136 +++++++++ .../test_passthrough_stream_cost_e2e.py | 68 +++++ .../test_service_tier_pricing_e2e.py | 116 ++++++++ 7 files changed, 817 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/quota_management/spend_tracking/cost_rows.py create mode 100644 tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py create mode 100644 tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py create mode 100644 tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py create mode 100644 tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 2dfa7adddea..5438ed8534a 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -44,3 +44,10 @@ - {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"} - {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"} - {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"} +- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"} +- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"} +- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"} +- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} +- {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} +- {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} +- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503)"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index 619f4dcacfe..fe93b13e0a4 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -216,10 +216,19 @@ class McpChatTool(BaseModel): allowed_tools: list[str] | None = None +class StreamOptions(BaseModel): + """OpenAI `stream_options`: `include_usage` asks for a final usage-only SSE + frame, which is where the proxy's `include_cost_in_streaming_usage` setting + injects `usage.cost`.""" + + include_usage: bool = True + + class ChatBody(BaseModel): model: str messages: list[ChatMessage] stream: bool = False + stream_options: StreamOptions | None = None max_tokens: int | None = None max_completion_tokens: int | None = None temperature: float | None = None @@ -322,6 +331,9 @@ class CompletionTokensDetails(BaseModel): class Usage(BaseModel): + """`cost` exists only on streaming usage frames from a proxy running with + `include_cost_in_streaming_usage: true`; providers never send it.""" + prompt_tokens: int | None = None completion_tokens: int | None = None total_tokens: int | None = None @@ -329,6 +341,7 @@ class Usage(BaseModel): cache_creation_input_tokens: int | None = None prompt_tokens_details: PromptTokensDetails | None = None completion_tokens_details: CompletionTokensDetails | None = None + cost: float | None = None class ChatResponse(BaseModel): @@ -449,9 +462,11 @@ class AnthropicMessagesResponse(BaseModel): for triage.""" model_config = ConfigDict(extra="allow") + id: str | None = None model: str | None = None content: list[AnthropicContentBlock] | None = None choices: list[ChatChoice] | None = None + usage: Usage | None = None class CountTokensResponse(BaseModel): @@ -716,8 +731,10 @@ class FineTuningJobsResponse(BaseModel): class LiteLLMParamsBody(BaseModel): """POST /model/new litellm_params: `model` is the only required field; `api_key` et al may be an `os.environ/FOO` reference the proxy resolves at call time. - `input_cost_per_token`/`output_cost_per_token` register a per-deployment custom - pricing override; left None (and dropped from the body) the deployment keeps the + The `*_cost_per_token` / `*_token_cost` fields register a per-deployment custom + pricing override (the cache and `_priority` rates only apply when both base + rates are set, which is what makes the proxy register the deployment's full + pricing entry); left None (and dropped from the body) the deployment keeps the backend's canonical rate.""" model: str @@ -744,6 +761,10 @@ class LiteLLMParamsBody(BaseModel): aws_external_id: str | None = None input_cost_per_token: float | None = None output_cost_per_token: float | None = None + cache_read_input_token_cost: float | None = None + cache_creation_input_token_cost: float | None = None + input_cost_per_token_priority: float | None = None + output_cost_per_token_priority: float | None = None extra_headers: dict[str, str] | None = None use_in_pass_through: bool | None = None complexity_router_config: dict[str, object] | None = None diff --git a/tests/e2e/quota_management/spend_tracking/cost_rows.py b/tests/e2e/quota_management/spend_tracking/cost_rows.py new file mode 100644 index 00000000000..87af54fe83f --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/cost_rows.py @@ -0,0 +1,204 @@ +"""Cost-accounting helpers for the spend-tracking suite: the /spend/logs row shape +that carries the per-component cost breakdown, a poll that waits for it, and the +builders the cache-pricing tests share. + +The shared SpendLogRow deliberately stays thin (most tests only read totals), so +the component-cost tests model the metadata they assert on here instead: +`metadata.cost_breakdown` (input/output/cache-read/cache-creation/reasoning costs +plus the service-tier pricing basis) and `metadata.additional_usage_values` (the +cache token counts the biller derived from the provider's usage). + +Determinism strategy: every test registers its own deployment with explicit custom +rates for each component it asserts on (`register_priced_model`), so expected cost +is exactly tokens-on-the-row times configured rate, immune to provider price +changes. The rates are chosen ~100x above canonical and distinct from one another, +so a component billed at the wrong rate can never accidentally match. + +OpenAI prompt caching is implicit and keyed on the exact token prefix, with a +1024-token minimum. `cacheable_prefix` builds a prefix whose first word is the +run's unique marker: unique marker = the whole prefix is novel (a fresh cache +write), same marker + different question = a cache read that still misses the +proxy's own response cache. How long the prefix has to be before the provider +actually reports a read varies by model, so callers pass `words` to suit theirs. + +Two facts about the recorded bill that the assertions here encode, because the +two surfaces disagree on purpose. On the spend row, `input_cost` is gross: it +already contains the cache-read and cache-creation costs, so the row's total is +input + output + tool-usage and the fresh-token cost is input minus the two cache +components. In the response headers, `x-litellm-response-cost-input` is net of +cache, which is what makes the component headers sum to the total. +""" + +import time +from collections.abc import Callable + +from pydantic import BaseModel, RootModel + +from e2e_config import unique_marker +from e2e_http import Success +from lifecycle import ResourceManager +from models import LiteLLMParamsBody, SpendLogsParams +from proxy_client import ProxyClient + + +class CostBreakdownRow(BaseModel): + input_cost: float | None = None + output_cost: float | None = None + cache_read_cost: float | None = None + cache_creation_cost: float | None = None + reasoning_cost: float | None = None + tool_usage_cost: float | None = None + total_cost: float | None = None + service_tier: str | None = None + + +class AdditionalUsageValues(BaseModel): + cache_read_input_tokens: int | None = None + cache_creation_input_tokens: int | None = None + + +class CostRowMetadata(BaseModel): + cost_breakdown: CostBreakdownRow | None = None + additional_usage_values: AdditionalUsageValues | None = None + + +class CostRow(BaseModel): + request_id: str | None = None + spend: float | None = None + prompt_tokens: int | None = None + completion_tokens: int | None = None + metadata: CostRowMetadata | None = None + + @property + def breakdown(self) -> CostBreakdownRow: + assert self.metadata and self.metadata.cost_breakdown, ( + f"spend row {self.request_id} landed without a cost breakdown" + ) + return self.metadata.cost_breakdown + + @property + def cache_read_tokens(self) -> int: + if self.metadata and self.metadata.additional_usage_values: + return self.metadata.additional_usage_values.cache_read_input_tokens or 0 + return 0 + + @property + def cache_creation_tokens(self) -> int: + if self.metadata and self.metadata.additional_usage_values: + return self.metadata.additional_usage_values.cache_creation_input_tokens or 0 + return 0 + + +class CostRows(RootModel[list[CostRow]]): + pass + + +def approx_equal(actual: float, expected: float) -> bool: + """Within 1% or 1e-9 absolute - spend math, not exact float identity.""" + return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2) + + +def assert_total_is_sum_of_components(row: CostRow) -> None: + """The row's total is input + output + tool usage. The cache components are + already inside the gross input cost, so adding them again would double-bill.""" + breakdown = row.breakdown + components = sum( + cost or 0.0 + for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost) + ) + assert breakdown.total_cost is not None and approx_equal(breakdown.total_cost, components), ( + f"total_cost {breakdown.total_cost} != input + output + tool usage ({components}): {breakdown}" + ) + assert row.spend is not None and approx_equal(row.spend, breakdown.total_cost), ( + f"row spend {row.spend} != breakdown total {breakdown.total_cost}" + ) + + +def assert_fresh_tokens_billed_at(row: CostRow, input_rate: float) -> None: + """Strip the cache components out of the gross input cost and what is left must + be the freshly-read tokens at the deployment's input rate.""" + breakdown = row.breakdown + fresh_tokens = (row.prompt_tokens or 0) - row.cache_read_tokens - row.cache_creation_tokens + fresh_cost = ( + (breakdown.input_cost or 0.0) + - (breakdown.cache_read_cost or 0.0) + - (breakdown.cache_creation_cost or 0.0) + ) + assert breakdown.input_cost is not None and approx_equal(fresh_cost, fresh_tokens * input_rate), ( + f"input_cost {breakdown.input_cost} less cache read {breakdown.cache_read_cost} and " + f"cache creation {breakdown.cache_creation_cost} leaves {fresh_cost}, not " + f"{fresh_tokens} fresh tokens * {input_rate} (prompt {row.prompt_tokens}, " + f"cache read {row.cache_read_tokens}, cache creation {row.cache_creation_tokens}); " + "cached tokens are being billed at the input rate" + ) + + +def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None: + """Poll /spend/logs for the call's row until it lands with a cost breakdown + (rows flush ~60s behind the call via proxy_batch_write_at); None on timeout.""" + deadline = time.monotonic() + proxy.poll_timeout + while time.monotonic() < deadline: + result = proxy.transport.get( + "/spend/logs", + headers=proxy.transport.master, + params=SpendLogsParams(request_id=request_id), + response_type=CostRows, + ) + match result: + case Success(data=data): + rows = data.root + case _: + rows = [] + for row in rows: + if row.metadata and row.metadata.cost_breakdown: + return row + time.sleep(proxy.poll_interval) + return None + + +def poll_cost_row_where( + proxy: ProxyClient, api_key: str, predicate: Callable[[CostRow], bool] +) -> CostRow | None: + """Poll the key's own /spend/logs until one of its rows carries a cost breakdown + the predicate accepts; None on timeout. For calls whose response id is not the + id the bill is filed under, which is how a user finds the row in the UI anyway.""" + deadline = time.monotonic() + proxy.poll_timeout + while time.monotonic() < deadline: + result = proxy.transport.get( + "/spend/logs", + headers=proxy.transport.master, + params=SpendLogsParams(api_key=api_key), + response_type=CostRows, + ) + match result: + case Success(data=data): + rows = data.root + case _: + rows = [] + for row in rows: + if row.metadata and row.metadata.cost_breakdown and predicate(row): + return row + time.sleep(proxy.poll_interval) + return None + + +def register_priced_model( + proxy: ProxyClient, + resources: ResourceManager, + name_prefix: str, + litellm_params: LiteLLMParamsBody, +) -> str: + """Register a deployment with explicit custom rates (deleted on teardown) and + return its unique model name.""" + model_name = f"{name_prefix}-{unique_marker()}" + model_id = proxy.create_model(model_name, litellm_params) + resources.defer(lambda: proxy.delete_model(model_id)) + return model_name + + +def cacheable_prefix(marker: str, *, words: int = 1200) -> str: + """A prompt prefix above OpenAI's 1024-token caching minimum whose identity is + fully determined by `marker` (it is the first word, and prefix caching matches + from token zero). Raise `words` for models that only report a cache read on a + substantially longer prefix.""" + return " ".join(marker if i == 0 else f"token{i:04d}" for i in range(words)) diff --git a/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py new file mode 100644 index 00000000000..ca5985fcda6 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py @@ -0,0 +1,263 @@ +"""Live e2e: prompt-cache token accounting bills each cache component at its own rate. + +Four regressions the gateway has shipped fixes for, pinned against real OpenAI +prompt caching (implicit, keyed on the token prefix). Every test registers its own +deployment with distinct custom rates for input / output / cache-read / +cache-creation, so the expected bill is exactly the row's token counts times the +configured rates and a component billed at the wrong rate can never pass: + +- cache writes: gpt-5.6's cache-write tokens must land on the spend row as + cache-creation tokens billed at the cache-creation rate, not silently at the + input rate (#34046) +- breakdown components: the row's metadata.cost_breakdown must itemize cache-read, + cache-creation, and reasoning costs, with reasoning a subset of output (#31686) +- streaming: a streamed call's reassembled usage must keep the cached-token detail + so cache reads bill at the cache-read discount, not full input price (#34812) +- /v1/messages bridge: a request served by a Responses-only OpenAI model crosses + the anthropic-messages -> Responses adapter and must keep its cache-read tokens + and their discounted billing (#34957) + +Each test drives the model that actually reports the component it bills, which is +not the same model throughout. gpt-5.6-luna reports cache-write tokens on every +call over the caching minimum and never reports a cache read, so it is the one +model that can prove cache-write billing and the one model that can never prove +cache-read billing. gpt-5.5 is the reverse: it reports cached tokens on the second +call and no cache writes at all. gpt-5.3-codex is Responses-only, which is what +forces the /v1/messages bridge, and it starts reporting cache reads once the +prefix is a few thousand tokens rather than one. + +OpenAI caching is best-effort, so each test retries with a fresh prefix (new +marker = brand-new cache identity) up to three times before failing; the prime and +measured calls share the prefix but differ in the trailing question, which defeats +the proxy's own response cache without touching the provider's prefix cache. +""" + +import pytest + +from cost_rows import ( + CostRow, + approx_equal, + assert_fresh_tokens_billed_at, + assert_total_is_sum_of_components, + cacheable_prefix, + poll_cost_row, + poll_cost_row_where, + register_priced_model, +) +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import AnthropicMessagesBody, ChatBody, ChatMessage, LiteLLMParamsBody +from pydantic import BaseModel +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +CACHE_WRITE_BACKEND = "openai/gpt-5.6-luna" +CACHE_READ_BACKEND = "openai/gpt-5.5" +BRIDGE_BACKEND = "openai/gpt-5.3-codex" +BRIDGE_PREFIX_WORDS = 3000 +OPENAI_API_KEY = "os.environ/OPENAI_API_KEY" +CACHE_ATTEMPTS = 3 + +INPUT_RATE = 4e-05 +OUTPUT_RATE = 8e-05 +CACHE_READ_RATE = 1e-05 +CACHE_WRITE_RATE = 5e-05 + +PRIME_QUESTION = "Reply with the single word ready." +REASONING_QUESTION = "Compute 47*83 - 19*7 step by step, then reply with just the final number." + + +class _StreamChunk(BaseModel): + id: str | None = None + + +def _cache_priced_params(backend: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=backend, + api_key=OPENAI_API_KEY, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + cache_read_input_token_cost=CACHE_READ_RATE, + cache_creation_input_token_cost=CACHE_WRITE_RATE, + ) + + +def _chat_body(model: str, content: str, *, stream: bool = False) -> ChatBody: + return ChatBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + stream=stream, + max_completion_tokens=4000, + ) + + +def _require_row(client: SpendClient, request_id: str) -> CostRow: + row = poll_cost_row(client.proxy, request_id) + assert row is not None, f"no spend row with a cost breakdown landed for {request_id}" + return row + + +def _assert_cache_read_billed(row: CostRow) -> None: + assert row.breakdown.cache_read_cost is not None and approx_equal( + row.breakdown.cache_read_cost, row.cache_read_tokens * CACHE_READ_RATE + ), ( + f"cache_read_cost {row.breakdown.cache_read_cost} != " + f"{row.cache_read_tokens} cached tokens * {CACHE_READ_RATE}" + ) + assert_fresh_tokens_billed_at(row, INPUT_RATE) + assert_total_is_sum_of_components(row) + + +class TestCacheCostAccounting: + @pytest.mark.covers("quota_management.spend_tracking.cache_write.bills_cache_creation_rate") + def test_cache_write_tokens_billed_at_cache_creation_rate( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "cache-write-priced", _cache_priced_params(CACHE_WRITE_BACKEND) + ) + + for _ in range(CACHE_ATTEMPTS): + prompt = f"{cacheable_prefix(unique_marker())}\n{PRIME_QUESTION}" + chat = unwrap(client.proxy.chat(scoped_key, _chat_body(model, prompt))) + assert chat.id, f"chat response carried no id: {chat}" + row = _require_row(client, chat.id) + if row.cache_creation_tokens > 0: + break + else: + pytest.fail( + f"OpenAI reported no cache-write tokens across {CACHE_ATTEMPTS} fresh " + "~2k-token prompts; the cache-write billing path was never exercised" + ) + + assert row.breakdown.cache_creation_cost is not None and approx_equal( + row.breakdown.cache_creation_cost, row.cache_creation_tokens * CACHE_WRITE_RATE + ), ( + f"cache_creation_cost {row.breakdown.cache_creation_cost} != " + f"{row.cache_creation_tokens} cache-write tokens * {CACHE_WRITE_RATE}" + ) + assert_fresh_tokens_billed_at(row, INPUT_RATE) + assert_total_is_sum_of_components(row) + + @pytest.mark.covers("quota_management.spend_tracking.cost_breakdown.reports_component_costs") + def test_cost_breakdown_reports_component_costs( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "breakdown-priced", _cache_priced_params(CACHE_READ_BACKEND) + ) + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker()) + unwrap(client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{PRIME_QUESTION}"))) + chat = unwrap( + client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{REASONING_QUESTION}")) + ) + assert chat.id, f"chat response carried no id: {chat}" + row = _require_row(client, chat.id) + if row.cache_read_tokens > 0: + break + else: + pytest.fail( + f"no cache read landed across {CACHE_ATTEMPTS} prime+read rounds; " + "the component-cost breakdown was never exercised with cached input" + ) + + usage = chat.usage + assert usage is not None and usage.completion_tokens_details is not None, ( + f"no completion token details on the measured call: {chat}" + ) + reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0 + assert reasoning_tokens > 0, f"the reasoning question produced no reasoning tokens: {usage}" + + breakdown = row.breakdown + assert breakdown.output_cost is not None and approx_equal( + breakdown.output_cost, (row.completion_tokens or 0) * OUTPUT_RATE + ), ( + f"output_cost {breakdown.output_cost} != " + f"{row.completion_tokens} completion tokens * {OUTPUT_RATE}" + ) + assert breakdown.reasoning_cost is not None and approx_equal( + breakdown.reasoning_cost, reasoning_tokens * OUTPUT_RATE + ), ( + f"reasoning_cost {breakdown.reasoning_cost} != " + f"{reasoning_tokens} reasoning tokens * {OUTPUT_RATE}" + ) + assert breakdown.reasoning_cost <= (breakdown.output_cost or 0.0) * 1.01, ( + f"reasoning_cost {breakdown.reasoning_cost} exceeds output_cost " + f"{breakdown.output_cost}; reasoning must be a subset of output" + ) + _assert_cache_read_billed(row) + + @pytest.mark.covers("quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate") + def test_streaming_cache_read_billed_at_cache_read_rate( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "stream-cache-priced", _cache_priced_params(CACHE_READ_BACKEND) + ) + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker()) + unwrap(client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{PRIME_QUESTION}"))) + result = client.proxy.chat_stream( + scoped_key, + _chat_body(model, f"{prefix}\nReply with the single word cached.", stream=True), + ) + assert result.ok and result.stream_events, ( + f"streamed chat failed (status {result.status_code}): {result.body[:300]}" + ) + stream_id = _StreamChunk.model_validate_json(result.stream_events[0]).id + assert stream_id, f"first stream chunk carried no id: {result.stream_events[0][:200]}" + row = _require_row(client, stream_id) + if row.cache_read_tokens > 0: + break + else: + pytest.fail( + f"no cache read landed across {CACHE_ATTEMPTS} prime+stream rounds; " + "streaming cache-read billing was never exercised" + ) + + _assert_cache_read_billed(row) + + @pytest.mark.covers("quota_management.spend_tracking.messages_bridge.keeps_cache_tokens") + def test_messages_bridge_keeps_cache_tokens( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, resources, "bridge-cache-priced", _cache_priced_params(BRIDGE_BACKEND) + ) + + def bridge_call(content: str) -> int: + response = unwrap( + client.proxy.messages( + scoped_key, + AnthropicMessagesBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_tokens=4000, + ), + ) + ) + assert response.usage is not None, f"bridged response carried no usage: {response}" + return response.usage.cache_read_input_tokens or 0 + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker(), words=BRIDGE_PREFIX_WORDS) + bridge_call(f"{prefix}\n{PRIME_QUESTION}") + if bridge_call(f"{prefix}\nReply with the single word bridged.") > 0: + break + else: + pytest.fail( + f"no cache read survived {CACHE_ATTEMPTS} bridged prime+read rounds; " + "cache tokens are not surviving the anthropic-messages -> Responses bridge" + ) + + row = poll_cost_row_where(client.proxy, scoped_key, lambda r: r.cache_read_tokens > 0) + assert row is not None, ( + "the bridged call reported cached tokens but no spend row for the key " + "recorded any; the cache tokens were dropped on the way to the bill" + ) + _assert_cache_read_billed(row) diff --git a/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py new file mode 100644 index 00000000000..203be611905 --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_cost_headers_e2e.py @@ -0,0 +1,136 @@ +"""Live e2e: the per-component x-litellm-response-cost-* headers keep their contract. + +Pins the header contract shipped in #36965: alongside the x-litellm-response-cost +total, every response carries the component costs (input, output, cache-read, +cache-creation, reasoning, tool-usage), where input covers only fresh tokens (the +cache components are subtracted out) so the components sum to the total, and +reasoning stays a subset of output. + +The deployment carries distinct custom rates per component, a prime call fills the +provider's prefix cache, and the measured call re-reads it, so the cache-read +header is exercised with a real nonzero value instead of passing vacuously. The +backend is gpt-5.5 because it reports cached tokens on the second call; the +gpt-5.6 line reports cache writes and never a read, which would leave the +cache-read header at zero forever. The raw-transport send is used because the +typed chat client validates bodies and drops headers. OpenAI caching is +best-effort, so the prime+measure round retries with a fresh prefix before +failing. +""" + +import pytest + +from cost_rows import approx_equal, cacheable_prefix, register_priced_model +from e2e_config import unique_marker +from e2e_http import StreamingResponse +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +BACKEND = "openai/gpt-5.5" +OPENAI_API_KEY = "os.environ/OPENAI_API_KEY" +CACHE_ATTEMPTS = 3 + +INPUT_RATE = 4e-05 +OUTPUT_RATE = 8e-05 +CACHE_READ_RATE = 1e-05 +CACHE_WRITE_RATE = 5e-05 + +COMPONENT_HEADERS = ( + "x-litellm-response-cost-input", + "x-litellm-response-cost-cache-read", + "x-litellm-response-cost-cache-creation", + "x-litellm-response-cost-output", + "x-litellm-response-cost-tool-usage", +) + + +def _header_cost(response: StreamingResponse, name: str) -> float: + value = response.headers.get(name) + return float(value) if value not in (None, "", "None") else 0.0 + + +class TestCostHeaders: + @pytest.mark.covers("quota_management.spend_tracking.cost_headers.additive_components") + def test_component_cost_headers_sum_to_total( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, + resources, + "header-priced", + LiteLLMParamsBody( + model=BACKEND, + api_key=OPENAI_API_KEY, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + cache_read_input_token_cost=CACHE_READ_RATE, + cache_creation_input_token_cost=CACHE_WRITE_RATE, + ), + ) + + def priced_call(content: str) -> StreamingResponse: + response = client.proxy.transport.send( + "/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=model, + messages=[ChatMessage(role="user", content=content)], + max_completion_tokens=4000, + ), + ) + assert response.ok, f"chat failed (status {response.status_code}): {response.body[:300]}" + return response + + for _ in range(CACHE_ATTEMPTS): + prefix = cacheable_prefix(unique_marker()) + priced_call(f"{prefix}\nReply with the single word ready.") + measured = priced_call(f"{prefix}\nReply with the single word measured.") + if _header_cost(measured, "x-litellm-response-cost-cache-read") > 0: + break + else: + pytest.fail( + f"no cache read landed across {CACHE_ATTEMPTS} prime+measure rounds; " + "the cache-read cost header was never exercised with a nonzero value" + ) + + total = measured.response_cost + assert total is not None and total > 0, ( + f"x-litellm-response-cost missing or zero: {measured.headers}" + ) + component_sum = sum(_header_cost(measured, name) for name in COMPONENT_HEADERS) + assert approx_equal(component_sum, total), ( + f"component headers sum to {component_sum}, not the total {total}: " + f"{ {name: measured.headers.get(name) for name in COMPONENT_HEADERS} }" + ) + + reasoning = _header_cost(measured, "x-litellm-response-cost-reasoning") + output = _header_cost(measured, "x-litellm-response-cost-output") + assert reasoning <= output * 1.01, ( + f"reasoning header {reasoning} exceeds output header {output}; " + "reasoning must be a subset of output" + ) + + usage = ChatResponse.model_validate_json(measured.body).usage + assert usage is not None, f"measured response carried no usage: {measured.body[:300]}" + cached_tokens = ( + usage.prompt_tokens_details.cached_tokens or 0 if usage.prompt_tokens_details else 0 + ) + cache_creation_tokens = usage.cache_creation_input_tokens or 0 + assert cached_tokens > 0, f"cache-read header nonzero but usage shows no cached tokens: {usage}" + assert approx_equal( + _header_cost(measured, "x-litellm-response-cost-cache-read"), + cached_tokens * CACHE_READ_RATE, + ), ( + f"cache-read header {measured.headers.get('x-litellm-response-cost-cache-read')} != " + f"{cached_tokens} cached tokens * {CACHE_READ_RATE}" + ) + fresh_tokens = (usage.prompt_tokens or 0) - cached_tokens - cache_creation_tokens + assert approx_equal( + _header_cost(measured, "x-litellm-response-cost-input"), fresh_tokens * INPUT_RATE + ), ( + f"input header {measured.headers.get('x-litellm-response-cost-input')} != " + f"{fresh_tokens} fresh tokens * {INPUT_RATE}; the input component is not " + "subtracting the cache components" + ) diff --git a/tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py b/tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py new file mode 100644 index 00000000000..4c3a2a4509f --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py @@ -0,0 +1,68 @@ +"""Live e2e: the /openai passthrough injects usage.cost into streaming usage frames. + +Pins #36503: with the proxy running `include_cost_in_streaming_usage: true`, a +streamed call through the provider passthrough surface must carry the computed +cost inside the final usage-only SSE frame, the same contract the native +/chat/completions stream has. Providers never send `cost` themselves, so a +nonzero value proves the proxy computed and injected it on the passthrough path. + +The row-side spend accounting for passthrough calls is covered elsewhere; this +test pins only the in-stream cost surface, which clients read without ever +touching /spend/logs. +""" + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from models import ChatBody, ChatMessage, StreamOptions, Usage +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +OPENAI_MODEL = "gpt-5.6-luna" + + +class _StreamFrame(BaseModel): + usage: Usage | None = None + + +class TestPassthroughStreamCost: + @pytest.mark.covers("quota_management.spend_tracking.passthrough_stream.injects_usage_cost") + def test_passthrough_stream_final_usage_frame_carries_cost( + self, client: SpendClient, scoped_key: str + ) -> None: + result = client.proxy.transport.send( + "/openai/v1/chat/completions", + headers=client.proxy.transport.bearer(scoped_key), + json=ChatBody( + model=OPENAI_MODEL, + messages=[ + ChatMessage( + role="user", + content=f"{unique_marker()} Reply with the single word passthrough.", + ) + ], + stream=True, + stream_options=StreamOptions(), + ), + stream=True, + ) + assert result.ok and result.stream_events, ( + f"passthrough stream failed (status {result.status_code}): {result.body[:300]}" + ) + + usage_frames = [ + frame.usage + for frame in (_StreamFrame.model_validate_json(event) for event in result.stream_events) + if frame.usage is not None + ] + assert usage_frames, ( + f"no usage frame in the passthrough stream despite stream_options.include_usage; " + f"last event: {result.stream_events[-1][:300]}" + ) + + final_usage = usage_frames[-1] + assert final_usage.cost is not None and final_usage.cost > 0, ( + f"final passthrough usage frame carries no injected cost: {final_usage}" + ) diff --git a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py new file mode 100644 index 00000000000..171c849fb4c --- /dev/null +++ b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py @@ -0,0 +1,116 @@ +"""Live e2e: a service_tier request bills every component at the tier's own rates. + +Pins the tier-billing fixes (#35923, #35925): a priority-tier call must price +input and output at the deployment's `*_priority` rates, including the reasoning +tokens inside output (the shipped bug billed reasoning at the default-tier rate), +and the spend row must record the tier the bill was computed on. + +The deployment carries custom base AND priority rates, each distinct, so a bill +computed from the wrong tier (or a mix) cannot match the expected numbers. The +prompt is a fresh unique marker per run, keeping cached tokens out of the math. +The response's own `service_tier` echo is asserted first: if OpenAI ever declined +priority processing and served the default tier, the test fails there instead of +producing a vacuous rate comparison. +""" + +import pytest + +from cost_rows import ( + approx_equal, + assert_fresh_tokens_billed_at, + assert_total_is_sum_of_components, + poll_cost_row, + register_priced_model, +) +from e2e_config import unique_marker +from e2e_http import unwrap +from lifecycle import ResourceManager +from models import ChatBody, ChatMessage, LiteLLMParamsBody +from spend_e2e_client import SpendClient + +pytestmark = pytest.mark.e2e + +BACKEND = "openai/gpt-5.6-luna" +OPENAI_API_KEY = "os.environ/OPENAI_API_KEY" + +INPUT_RATE = 4e-05 +OUTPUT_RATE = 8e-05 +PRIORITY_INPUT_RATE = 6e-05 +PRIORITY_OUTPUT_RATE = 1.6e-04 + + +class TestServiceTierPricing: + @pytest.mark.covers("quota_management.spend_tracking.service_tier.bills_tier_rates") + def test_priority_tier_bills_priority_rates( + self, client: SpendClient, resources: ResourceManager, scoped_key: str + ) -> None: + model = register_priced_model( + client.proxy, + resources, + "tier-priced", + LiteLLMParamsBody( + model=BACKEND, + api_key=OPENAI_API_KEY, + input_cost_per_token=INPUT_RATE, + output_cost_per_token=OUTPUT_RATE, + input_cost_per_token_priority=PRIORITY_INPUT_RATE, + output_cost_per_token_priority=PRIORITY_OUTPUT_RATE, + ), + ) + + chat = unwrap( + client.proxy.chat( + scoped_key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=( + f"{unique_marker()} Compute 47*83 - 19*7 step by step, " + "then reply with just the final number." + ), + ) + ], + max_completion_tokens=4000, + service_tier="priority", + ), + ) + ) + assert chat.service_tier == "priority", ( + f"OpenAI served tier {chat.service_tier!r} instead of priority; " + "tier billing was never exercised" + ) + assert chat.id, f"chat response carried no id: {chat}" + + row = poll_cost_row(client.proxy, chat.id) + assert row is not None, f"no spend row with a cost breakdown landed for {chat.id}" + breakdown = row.breakdown + + assert breakdown.service_tier == "priority", ( + f"the bill records pricing basis {breakdown.service_tier!r}, not priority" + ) + + assert_fresh_tokens_billed_at(row, PRIORITY_INPUT_RATE) + assert breakdown.output_cost is not None and approx_equal( + breakdown.output_cost, (row.completion_tokens or 0) * PRIORITY_OUTPUT_RATE + ), ( + f"output_cost {breakdown.output_cost} != {row.completion_tokens} tokens * priority rate " + f"{PRIORITY_OUTPUT_RATE} (base rate would give {(row.completion_tokens or 0) * OUTPUT_RATE})" + ) + + usage = chat.usage + assert usage is not None and usage.completion_tokens_details is not None, ( + f"no completion token details on the priority call: {chat}" + ) + reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0 + assert reasoning_tokens > 0, f"the reasoning question produced no reasoning tokens: {usage}" + assert breakdown.reasoning_cost is not None and approx_equal( + breakdown.reasoning_cost, reasoning_tokens * PRIORITY_OUTPUT_RATE + ), ( + f"reasoning_cost {breakdown.reasoning_cost} != {reasoning_tokens} reasoning tokens * " + f"priority rate {PRIORITY_OUTPUT_RATE} (the default-tier rate would give " + f"{reasoning_tokens * OUTPUT_RATE})" + ) + + assert_total_is_sum_of_components(row) From aa8e7278e3d27a483ef8039995bce649dc6c0d88 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:08:05 -0700 Subject: [PATCH 2/4] test(e2e): drop the passthrough streaming-cost test, it needs a config flag The final streaming usage frame only carries usage.cost when the proxy runs with litellm_settings.include_cost_in_streaming_usage: true, and that flag is readable only off the module-level litellm setting. There is no header, key, or management route that turns it on per request, so a test cannot ask the shared e2e proxy for it, and the proxy's config does not live in this repo. The registry row stays as an uncovered gap with the reason recorded, rather than being deleted, so the behavior is still on the list of things we want covered once the gateway config is reachable. The StreamOptions model, ChatBody.stream_options, Usage.cost, and AnthropicMessagesResponse.id existed only for that test, so they go with it. --- .../coverage_registry/quota_management.yaml | 2 +- tests/e2e/models.py | 14 ---- .../test_passthrough_stream_cost_e2e.py | 68 ------------------- 3 files changed, 1 insertion(+), 83 deletions(-) delete mode 100644 tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 5438ed8534a..98a45eefb2d 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -50,4 +50,4 @@ - {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} -- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503)"} +- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e proxy's config is not in this repo"} diff --git a/tests/e2e/models.py b/tests/e2e/models.py index fe93b13e0a4..f1d5733253a 100644 --- a/tests/e2e/models.py +++ b/tests/e2e/models.py @@ -216,19 +216,10 @@ class McpChatTool(BaseModel): allowed_tools: list[str] | None = None -class StreamOptions(BaseModel): - """OpenAI `stream_options`: `include_usage` asks for a final usage-only SSE - frame, which is where the proxy's `include_cost_in_streaming_usage` setting - injects `usage.cost`.""" - - include_usage: bool = True - - class ChatBody(BaseModel): model: str messages: list[ChatMessage] stream: bool = False - stream_options: StreamOptions | None = None max_tokens: int | None = None max_completion_tokens: int | None = None temperature: float | None = None @@ -331,9 +322,6 @@ class CompletionTokensDetails(BaseModel): class Usage(BaseModel): - """`cost` exists only on streaming usage frames from a proxy running with - `include_cost_in_streaming_usage: true`; providers never send it.""" - prompt_tokens: int | None = None completion_tokens: int | None = None total_tokens: int | None = None @@ -341,7 +329,6 @@ class Usage(BaseModel): cache_creation_input_tokens: int | None = None prompt_tokens_details: PromptTokensDetails | None = None completion_tokens_details: CompletionTokensDetails | None = None - cost: float | None = None class ChatResponse(BaseModel): @@ -462,7 +449,6 @@ class AnthropicMessagesResponse(BaseModel): for triage.""" model_config = ConfigDict(extra="allow") - id: str | None = None model: str | None = None content: list[AnthropicContentBlock] | None = None choices: list[ChatChoice] | None = None diff --git a/tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py b/tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py deleted file mode 100644 index 4c3a2a4509f..00000000000 --- a/tests/e2e/quota_management/spend_tracking/test_passthrough_stream_cost_e2e.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Live e2e: the /openai passthrough injects usage.cost into streaming usage frames. - -Pins #36503: with the proxy running `include_cost_in_streaming_usage: true`, a -streamed call through the provider passthrough surface must carry the computed -cost inside the final usage-only SSE frame, the same contract the native -/chat/completions stream has. Providers never send `cost` themselves, so a -nonzero value proves the proxy computed and injected it on the passthrough path. - -The row-side spend accounting for passthrough calls is covered elsewhere; this -test pins only the in-stream cost surface, which clients read without ever -touching /spend/logs. -""" - -import pytest -from pydantic import BaseModel - -from e2e_config import unique_marker -from models import ChatBody, ChatMessage, StreamOptions, Usage -from spend_e2e_client import SpendClient - -pytestmark = pytest.mark.e2e - -OPENAI_MODEL = "gpt-5.6-luna" - - -class _StreamFrame(BaseModel): - usage: Usage | None = None - - -class TestPassthroughStreamCost: - @pytest.mark.covers("quota_management.spend_tracking.passthrough_stream.injects_usage_cost") - def test_passthrough_stream_final_usage_frame_carries_cost( - self, client: SpendClient, scoped_key: str - ) -> None: - result = client.proxy.transport.send( - "/openai/v1/chat/completions", - headers=client.proxy.transport.bearer(scoped_key), - json=ChatBody( - model=OPENAI_MODEL, - messages=[ - ChatMessage( - role="user", - content=f"{unique_marker()} Reply with the single word passthrough.", - ) - ], - stream=True, - stream_options=StreamOptions(), - ), - stream=True, - ) - assert result.ok and result.stream_events, ( - f"passthrough stream failed (status {result.status_code}): {result.body[:300]}" - ) - - usage_frames = [ - frame.usage - for frame in (_StreamFrame.model_validate_json(event) for event in result.stream_events) - if frame.usage is not None - ] - assert usage_frames, ( - f"no usage frame in the passthrough stream despite stream_options.include_usage; " - f"last event: {result.stream_events[-1][:300]}" - ) - - final_usage = usage_frames[-1] - assert final_usage.cost is not None and final_usage.cost > 0, ( - f"final passthrough usage frame carries no injected cost: {final_usage}" - ) From 69278ae37bba01a5ad19a83273719794c5b47c91 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 03:25:31 -0700 Subject: [PATCH 3/4] docs(e2e): correct the passthrough-stream registry row's uncovered reason --- tests/e2e/coverage_registry/quota_management.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/coverage_registry/quota_management.yaml b/tests/e2e/coverage_registry/quota_management.yaml index 98a45eefb2d..e4b7e755bd0 100644 --- a/tests/e2e/coverage_registry/quota_management.yaml +++ b/tests/e2e/coverage_registry/quota_management.yaml @@ -50,4 +50,4 @@ - {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"} - {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"} - {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"} -- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e proxy's config is not in this repo"} +- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503). Uncovered: the flag is only settable in litellm_settings, and the shared e2e stack does not turn it on yet"} From 975a6806c318976efd1d9adc6ee44aed4d8ff8d4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 20 Aug 2026 04:11:08 -0700 Subject: [PATCH 4/4] test(e2e): request reasoning explicitly on the reasoning-cost assertions The two tests that assert on reasoning cost read reasoning_tokens off the response and required it to be nonzero, without ever asking the model to reason. Both now send reasoning_effort, so the assertion rests on a parameter the test sets rather than on the model's default behavior. The cache-breakdown test sends it on its prime call too: OpenAI's prefix cache keys on the reasoning setting as well as the tokens, so priming at a different effort never produces a read. --- .../test_cache_cost_accounting_e2e.py | 30 +++++++++++++++++-- .../test_service_tier_pricing_e2e.py | 7 ++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py index ca5985fcda6..c50ec3d902f 100644 --- a/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_cache_cost_accounting_e2e.py @@ -30,6 +30,12 @@ OpenAI caching is best-effort, so each test retries with a fresh prefix (new marker = brand-new cache identity) up to three times before failing; the prime and measured calls share the prefix but differ in the trailing question, which defeats the proxy's own response cache without touching the provider's prefix cache. + +The test that asserts on reasoning cost requests reasoning explicitly with +`reasoning_effort`, so that assertion rests on a parameter the test sets rather +than on whatever the model happens to do by default. Its prime call carries the +same value: OpenAI's prefix cache keys on the reasoning setting as well as the +tokens, so a prime at a different effort never produces a read. """ import pytest @@ -67,6 +73,7 @@ CACHE_WRITE_RATE = 5e-05 PRIME_QUESTION = "Reply with the single word ready." REASONING_QUESTION = "Compute 47*83 - 19*7 step by step, then reply with just the final number." +REASONING_EFFORT = "high" class _StreamChunk(BaseModel): @@ -84,12 +91,15 @@ def _cache_priced_params(backend: str) -> LiteLLMParamsBody: ) -def _chat_body(model: str, content: str, *, stream: bool = False) -> ChatBody: +def _chat_body( + model: str, content: str, *, stream: bool = False, reasoning_effort: str | None = None +) -> ChatBody: return ChatBody( model=model, messages=[ChatMessage(role="user", content=content)], stream=stream, max_completion_tokens=4000, + reasoning_effort=reasoning_effort, ) @@ -151,9 +161,23 @@ class TestCacheCostAccounting: for _ in range(CACHE_ATTEMPTS): prefix = cacheable_prefix(unique_marker()) - unwrap(client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{PRIME_QUESTION}"))) + unwrap( + client.proxy.chat( + scoped_key, + _chat_body( + model, f"{prefix}\n{PRIME_QUESTION}", reasoning_effort=REASONING_EFFORT + ), + ) + ) chat = unwrap( - client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{REASONING_QUESTION}")) + client.proxy.chat( + scoped_key, + _chat_body( + model, + f"{prefix}\n{REASONING_QUESTION}", + reasoning_effort=REASONING_EFFORT, + ), + ) ) assert chat.id, f"chat response carried no id: {chat}" row = _require_row(client, chat.id) diff --git a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py index 171c849fb4c..770c5699b4e 100644 --- a/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py +++ b/tests/e2e/quota_management/spend_tracking/test_service_tier_pricing_e2e.py @@ -10,7 +10,9 @@ computed from the wrong tier (or a mix) cannot match the expected numbers. The prompt is a fresh unique marker per run, keeping cached tokens out of the math. The response's own `service_tier` echo is asserted first: if OpenAI ever declined priority processing and served the default tier, the test fails there instead of -producing a vacuous rate comparison. +producing a vacuous rate comparison. Reasoning is requested explicitly with +`reasoning_effort`, so the reasoning-rate assertion rests on a parameter the test +sets rather than on whatever the model happens to do by default. """ import pytest @@ -38,6 +40,8 @@ OUTPUT_RATE = 8e-05 PRIORITY_INPUT_RATE = 6e-05 PRIORITY_OUTPUT_RATE = 1.6e-04 +REASONING_EFFORT = "high" + class TestServiceTierPricing: @pytest.mark.covers("quota_management.spend_tracking.service_tier.bills_tier_rates") @@ -74,6 +78,7 @@ class TestServiceTierPricing: ], max_completion_tokens=4000, service_tier="priority", + reasoning_effort=REASONING_EFFORT, ), ) )