mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
test(e2e): pin prompt-cache, service-tier, and cost-header billing
Seven live e2e tests covering cost-tracking regressions that currently ship unnoticed: cache-write tokens billed at the cache-creation rate (#34046), per-component cost_breakdown on the spend row (#31686), cache reads billed at the cache-read discount on streamed calls (#34812), cache tokens surviving the anthropic-messages to Responses bridge (#34957), priority-tier rates applied to input, output and reasoning (#35923, #35925), the per-component response cost headers summing to the total (#36965), and cost injected into the final usage frame of an /openai passthrough stream (#36503). Every test registers its own deployment with a distinct custom rate per component, so a component billed at the wrong rate cannot pass. The shared helpers in cost_rows.py encode the one thing the two surfaces disagree on: the spend row's input_cost is gross of cache while the response's cost-input header is net of it.
This commit is contained in:
mateo-berri
parent
0b374541bb
commit
bcb6a6eaab
7 changed files with 817 additions and 2 deletions
|
|
@ -44,3 +44,10 @@
|
|||
- {id: quota_management.spend_tracking.failure.writes_failure_row, module: quota_management, tier: P1, behavior: spend_tracking, variant: failure, assertions: [writes_failure_row], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_log_error_logger.py", rationale: "A failed call writes a failure-status spend row"}
|
||||
- {id: quota_management.spend_tracking.spend_calculate.returns_cost, module: quota_management, tier: P2, behavior: spend_tracking, variant: spend_calculate, assertions: [returns_cost], exercised_on: [spend_calculate], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "/spend/calculate prices a hypothetical request at nonzero cost"}
|
||||
- {id: quota_management.spend_tracking.pagination.keeps_total, module: quota_management, tier: P2, behavior: spend_tracking, variant: pagination, assertions: [keeps_total], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_management_endpoints.py", rationale: "Spend-logs v2 pagination caps page size without losing the total"}
|
||||
- {id: quota_management.spend_tracking.cache_write.bills_cache_creation_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: cache_write, assertions: [bills_cache_creation_rate], exercised_on: [chat_completions], source: "litellm_core_utils/llm_cost_calc/utils.py", rationale: "OpenAI cache-write tokens land on the spend row as cache-creation tokens billed at the cache-creation rate, not silently at the input rate (#34046)"}
|
||||
- {id: quota_management.spend_tracking.cost_breakdown.reports_component_costs, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_breakdown, assertions: [reports_component_costs], exercised_on: [chat_completions], source: "proxy/spend_tracking/spend_tracking_utils.py", rationale: "The spend row's metadata.cost_breakdown itemizes cache-read, cache-creation, output, and reasoning costs at the deployment's own rates and they sum to the row's spend (#31686)"}
|
||||
- {id: quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate, module: quota_management, tier: P1, behavior: spend_tracking, variant: stream_cache_read, assertions: [bills_cache_read_rate], exercised_on: [chat_completions], source: "litellm_core_utils/streaming_chunk_builder_utils.py", rationale: "A streamed call's reassembled usage keeps the cached-token detail so cache reads bill at the cache-read discount, not full input price (#34812)"}
|
||||
- {id: quota_management.spend_tracking.messages_bridge.keeps_cache_tokens, module: quota_management, tier: P1, behavior: spend_tracking, variant: messages_bridge, assertions: [keeps_cache_tokens], exercised_on: [messages], source: "llms/anthropic/experimental_pass_through/responses_adapters/handler.py", rationale: "A /v1/messages request served by a Responses-only OpenAI model keeps its cache-read tokens and their discounted billing across the bridge (#34957)"}
|
||||
- {id: quota_management.spend_tracking.service_tier.bills_tier_rates, module: quota_management, tier: P1, behavior: spend_tracking, variant: service_tier, assertions: [bills_tier_rates], exercised_on: [chat_completions], source: "cost_calculator.py", rationale: "A priority service_tier call bills input, output, and reasoning at the deployment's *_priority rates and records the tier on the row (#35923, #35925)"}
|
||||
- {id: quota_management.spend_tracking.cost_headers.additive_components, module: quota_management, tier: P1, behavior: spend_tracking, variant: cost_headers, assertions: [additive_components], exercised_on: [chat_completions], source: "proxy/common_request_processing.py", rationale: "The x-litellm-response-cost-* component headers sum to the total, input covers only fresh tokens, and reasoning stays a subset of output (#36965)"}
|
||||
- {id: quota_management.spend_tracking.passthrough_stream.injects_usage_cost, module: quota_management, tier: P1, behavior: spend_tracking, variant: passthrough_stream, assertions: [injects_usage_cost], exercised_on: [openai_passthrough], source: "proxy/pass_through_endpoints/streaming_handler.py", rationale: "With include_cost_in_streaming_usage on, the /openai passthrough's final streaming usage frame carries the proxy-computed cost (#36503)"}
|
||||
|
|
|
|||
|
|
@ -216,10 +216,19 @@ class McpChatTool(BaseModel):
|
|||
allowed_tools: list[str] | None = None
|
||||
|
||||
|
||||
class StreamOptions(BaseModel):
|
||||
"""OpenAI `stream_options`: `include_usage` asks for a final usage-only SSE
|
||||
frame, which is where the proxy's `include_cost_in_streaming_usage` setting
|
||||
injects `usage.cost`."""
|
||||
|
||||
include_usage: bool = True
|
||||
|
||||
|
||||
class ChatBody(BaseModel):
|
||||
model: str
|
||||
messages: list[ChatMessage]
|
||||
stream: bool = False
|
||||
stream_options: StreamOptions | None = None
|
||||
max_tokens: int | None = None
|
||||
max_completion_tokens: int | None = None
|
||||
temperature: float | None = None
|
||||
|
|
@ -322,6 +331,9 @@ class CompletionTokensDetails(BaseModel):
|
|||
|
||||
|
||||
class Usage(BaseModel):
|
||||
"""`cost` exists only on streaming usage frames from a proxy running with
|
||||
`include_cost_in_streaming_usage: true`; providers never send it."""
|
||||
|
||||
prompt_tokens: int | None = None
|
||||
completion_tokens: int | None = None
|
||||
total_tokens: int | None = None
|
||||
|
|
@ -329,6 +341,7 @@ class Usage(BaseModel):
|
|||
cache_creation_input_tokens: int | None = None
|
||||
prompt_tokens_details: PromptTokensDetails | None = None
|
||||
completion_tokens_details: CompletionTokensDetails | None = None
|
||||
cost: float | None = None
|
||||
|
||||
|
||||
class ChatResponse(BaseModel):
|
||||
|
|
@ -449,9 +462,11 @@ class AnthropicMessagesResponse(BaseModel):
|
|||
for triage."""
|
||||
|
||||
model_config = ConfigDict(extra="allow")
|
||||
id: str | None = None
|
||||
model: str | None = None
|
||||
content: list[AnthropicContentBlock] | None = None
|
||||
choices: list[ChatChoice] | None = None
|
||||
usage: Usage | None = None
|
||||
|
||||
|
||||
class CountTokensResponse(BaseModel):
|
||||
|
|
@ -716,8 +731,10 @@ class FineTuningJobsResponse(BaseModel):
|
|||
class LiteLLMParamsBody(BaseModel):
|
||||
"""POST /model/new litellm_params: `model` is the only required field; `api_key`
|
||||
et al may be an `os.environ/FOO` reference the proxy resolves at call time.
|
||||
`input_cost_per_token`/`output_cost_per_token` register a per-deployment custom
|
||||
pricing override; left None (and dropped from the body) the deployment keeps the
|
||||
The `*_cost_per_token` / `*_token_cost` fields register a per-deployment custom
|
||||
pricing override (the cache and `_priority` rates only apply when both base
|
||||
rates are set, which is what makes the proxy register the deployment's full
|
||||
pricing entry); left None (and dropped from the body) the deployment keeps the
|
||||
backend's canonical rate."""
|
||||
|
||||
model: str
|
||||
|
|
@ -744,6 +761,10 @@ class LiteLLMParamsBody(BaseModel):
|
|||
aws_external_id: str | None = None
|
||||
input_cost_per_token: float | None = None
|
||||
output_cost_per_token: float | None = None
|
||||
cache_read_input_token_cost: float | None = None
|
||||
cache_creation_input_token_cost: float | None = None
|
||||
input_cost_per_token_priority: float | None = None
|
||||
output_cost_per_token_priority: float | None = None
|
||||
extra_headers: dict[str, str] | None = None
|
||||
use_in_pass_through: bool | None = None
|
||||
complexity_router_config: dict[str, object] | None = None
|
||||
|
|
|
|||
204
tests/e2e/quota_management/spend_tracking/cost_rows.py
Normal file
204
tests/e2e/quota_management/spend_tracking/cost_rows.py
Normal file
|
|
@ -0,0 +1,204 @@
|
|||
"""Cost-accounting helpers for the spend-tracking suite: the /spend/logs row shape
|
||||
that carries the per-component cost breakdown, a poll that waits for it, and the
|
||||
builders the cache-pricing tests share.
|
||||
|
||||
The shared SpendLogRow deliberately stays thin (most tests only read totals), so
|
||||
the component-cost tests model the metadata they assert on here instead:
|
||||
`metadata.cost_breakdown` (input/output/cache-read/cache-creation/reasoning costs
|
||||
plus the service-tier pricing basis) and `metadata.additional_usage_values` (the
|
||||
cache token counts the biller derived from the provider's usage).
|
||||
|
||||
Determinism strategy: every test registers its own deployment with explicit custom
|
||||
rates for each component it asserts on (`register_priced_model`), so expected cost
|
||||
is exactly tokens-on-the-row times configured rate, immune to provider price
|
||||
changes. The rates are chosen ~100x above canonical and distinct from one another,
|
||||
so a component billed at the wrong rate can never accidentally match.
|
||||
|
||||
OpenAI prompt caching is implicit and keyed on the exact token prefix, with a
|
||||
1024-token minimum. `cacheable_prefix` builds a prefix whose first word is the
|
||||
run's unique marker: unique marker = the whole prefix is novel (a fresh cache
|
||||
write), same marker + different question = a cache read that still misses the
|
||||
proxy's own response cache. How long the prefix has to be before the provider
|
||||
actually reports a read varies by model, so callers pass `words` to suit theirs.
|
||||
|
||||
Two facts about the recorded bill that the assertions here encode, because the
|
||||
two surfaces disagree on purpose. On the spend row, `input_cost` is gross: it
|
||||
already contains the cache-read and cache-creation costs, so the row's total is
|
||||
input + output + tool-usage and the fresh-token cost is input minus the two cache
|
||||
components. In the response headers, `x-litellm-response-cost-input` is net of
|
||||
cache, which is what makes the component headers sum to the total.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
from pydantic import BaseModel, RootModel
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import Success
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, SpendLogsParams
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
|
||||
class CostBreakdownRow(BaseModel):
|
||||
input_cost: float | None = None
|
||||
output_cost: float | None = None
|
||||
cache_read_cost: float | None = None
|
||||
cache_creation_cost: float | None = None
|
||||
reasoning_cost: float | None = None
|
||||
tool_usage_cost: float | None = None
|
||||
total_cost: float | None = None
|
||||
service_tier: str | None = None
|
||||
|
||||
|
||||
class AdditionalUsageValues(BaseModel):
|
||||
cache_read_input_tokens: int | None = None
|
||||
cache_creation_input_tokens: int | None = None
|
||||
|
||||
|
||||
class CostRowMetadata(BaseModel):
|
||||
cost_breakdown: CostBreakdownRow | None = None
|
||||
additional_usage_values: AdditionalUsageValues | None = None
|
||||
|
||||
|
||||
class CostRow(BaseModel):
|
||||
request_id: str | None = None
|
||||
spend: float | None = None
|
||||
prompt_tokens: int | None = None
|
||||
completion_tokens: int | None = None
|
||||
metadata: CostRowMetadata | None = None
|
||||
|
||||
@property
|
||||
def breakdown(self) -> CostBreakdownRow:
|
||||
assert self.metadata and self.metadata.cost_breakdown, (
|
||||
f"spend row {self.request_id} landed without a cost breakdown"
|
||||
)
|
||||
return self.metadata.cost_breakdown
|
||||
|
||||
@property
|
||||
def cache_read_tokens(self) -> int:
|
||||
if self.metadata and self.metadata.additional_usage_values:
|
||||
return self.metadata.additional_usage_values.cache_read_input_tokens or 0
|
||||
return 0
|
||||
|
||||
@property
|
||||
def cache_creation_tokens(self) -> int:
|
||||
if self.metadata and self.metadata.additional_usage_values:
|
||||
return self.metadata.additional_usage_values.cache_creation_input_tokens or 0
|
||||
return 0
|
||||
|
||||
|
||||
class CostRows(RootModel[list[CostRow]]):
|
||||
pass
|
||||
|
||||
|
||||
def approx_equal(actual: float, expected: float) -> bool:
|
||||
"""Within 1% or 1e-9 absolute - spend math, not exact float identity."""
|
||||
return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2)
|
||||
|
||||
|
||||
def assert_total_is_sum_of_components(row: CostRow) -> None:
|
||||
"""The row's total is input + output + tool usage. The cache components are
|
||||
already inside the gross input cost, so adding them again would double-bill."""
|
||||
breakdown = row.breakdown
|
||||
components = sum(
|
||||
cost or 0.0
|
||||
for cost in (breakdown.input_cost, breakdown.output_cost, breakdown.tool_usage_cost)
|
||||
)
|
||||
assert breakdown.total_cost is not None and approx_equal(breakdown.total_cost, components), (
|
||||
f"total_cost {breakdown.total_cost} != input + output + tool usage ({components}): {breakdown}"
|
||||
)
|
||||
assert row.spend is not None and approx_equal(row.spend, breakdown.total_cost), (
|
||||
f"row spend {row.spend} != breakdown total {breakdown.total_cost}"
|
||||
)
|
||||
|
||||
|
||||
def assert_fresh_tokens_billed_at(row: CostRow, input_rate: float) -> None:
|
||||
"""Strip the cache components out of the gross input cost and what is left must
|
||||
be the freshly-read tokens at the deployment's input rate."""
|
||||
breakdown = row.breakdown
|
||||
fresh_tokens = (row.prompt_tokens or 0) - row.cache_read_tokens - row.cache_creation_tokens
|
||||
fresh_cost = (
|
||||
(breakdown.input_cost or 0.0)
|
||||
- (breakdown.cache_read_cost or 0.0)
|
||||
- (breakdown.cache_creation_cost or 0.0)
|
||||
)
|
||||
assert breakdown.input_cost is not None and approx_equal(fresh_cost, fresh_tokens * input_rate), (
|
||||
f"input_cost {breakdown.input_cost} less cache read {breakdown.cache_read_cost} and "
|
||||
f"cache creation {breakdown.cache_creation_cost} leaves {fresh_cost}, not "
|
||||
f"{fresh_tokens} fresh tokens * {input_rate} (prompt {row.prompt_tokens}, "
|
||||
f"cache read {row.cache_read_tokens}, cache creation {row.cache_creation_tokens}); "
|
||||
"cached tokens are being billed at the input rate"
|
||||
)
|
||||
|
||||
|
||||
def poll_cost_row(proxy: ProxyClient, request_id: str) -> CostRow | None:
|
||||
"""Poll /spend/logs for the call's row until it lands with a cost breakdown
|
||||
(rows flush ~60s behind the call via proxy_batch_write_at); None on timeout."""
|
||||
deadline = time.monotonic() + proxy.poll_timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = proxy.transport.get(
|
||||
"/spend/logs",
|
||||
headers=proxy.transport.master,
|
||||
params=SpendLogsParams(request_id=request_id),
|
||||
response_type=CostRows,
|
||||
)
|
||||
match result:
|
||||
case Success(data=data):
|
||||
rows = data.root
|
||||
case _:
|
||||
rows = []
|
||||
for row in rows:
|
||||
if row.metadata and row.metadata.cost_breakdown:
|
||||
return row
|
||||
time.sleep(proxy.poll_interval)
|
||||
return None
|
||||
|
||||
|
||||
def poll_cost_row_where(
|
||||
proxy: ProxyClient, api_key: str, predicate: Callable[[CostRow], bool]
|
||||
) -> CostRow | None:
|
||||
"""Poll the key's own /spend/logs until one of its rows carries a cost breakdown
|
||||
the predicate accepts; None on timeout. For calls whose response id is not the
|
||||
id the bill is filed under, which is how a user finds the row in the UI anyway."""
|
||||
deadline = time.monotonic() + proxy.poll_timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = proxy.transport.get(
|
||||
"/spend/logs",
|
||||
headers=proxy.transport.master,
|
||||
params=SpendLogsParams(api_key=api_key),
|
||||
response_type=CostRows,
|
||||
)
|
||||
match result:
|
||||
case Success(data=data):
|
||||
rows = data.root
|
||||
case _:
|
||||
rows = []
|
||||
for row in rows:
|
||||
if row.metadata and row.metadata.cost_breakdown and predicate(row):
|
||||
return row
|
||||
time.sleep(proxy.poll_interval)
|
||||
return None
|
||||
|
||||
|
||||
def register_priced_model(
|
||||
proxy: ProxyClient,
|
||||
resources: ResourceManager,
|
||||
name_prefix: str,
|
||||
litellm_params: LiteLLMParamsBody,
|
||||
) -> str:
|
||||
"""Register a deployment with explicit custom rates (deleted on teardown) and
|
||||
return its unique model name."""
|
||||
model_name = f"{name_prefix}-{unique_marker()}"
|
||||
model_id = proxy.create_model(model_name, litellm_params)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
return model_name
|
||||
|
||||
|
||||
def cacheable_prefix(marker: str, *, words: int = 1200) -> str:
|
||||
"""A prompt prefix above OpenAI's 1024-token caching minimum whose identity is
|
||||
fully determined by `marker` (it is the first word, and prefix caching matches
|
||||
from token zero). Raise `words` for models that only report a cache read on a
|
||||
substantially longer prefix."""
|
||||
return " ".join(marker if i == 0 else f"token{i:04d}" for i in range(words))
|
||||
|
|
@ -0,0 +1,263 @@
|
|||
"""Live e2e: prompt-cache token accounting bills each cache component at its own rate.
|
||||
|
||||
Four regressions the gateway has shipped fixes for, pinned against real OpenAI
|
||||
prompt caching (implicit, keyed on the token prefix). Every test registers its own
|
||||
deployment with distinct custom rates for input / output / cache-read /
|
||||
cache-creation, so the expected bill is exactly the row's token counts times the
|
||||
configured rates and a component billed at the wrong rate can never pass:
|
||||
|
||||
- cache writes: gpt-5.6's cache-write tokens must land on the spend row as
|
||||
cache-creation tokens billed at the cache-creation rate, not silently at the
|
||||
input rate (#34046)
|
||||
- breakdown components: the row's metadata.cost_breakdown must itemize cache-read,
|
||||
cache-creation, and reasoning costs, with reasoning a subset of output (#31686)
|
||||
- streaming: a streamed call's reassembled usage must keep the cached-token detail
|
||||
so cache reads bill at the cache-read discount, not full input price (#34812)
|
||||
- /v1/messages bridge: a request served by a Responses-only OpenAI model crosses
|
||||
the anthropic-messages -> Responses adapter and must keep its cache-read tokens
|
||||
and their discounted billing (#34957)
|
||||
|
||||
Each test drives the model that actually reports the component it bills, which is
|
||||
not the same model throughout. gpt-5.6-luna reports cache-write tokens on every
|
||||
call over the caching minimum and never reports a cache read, so it is the one
|
||||
model that can prove cache-write billing and the one model that can never prove
|
||||
cache-read billing. gpt-5.5 is the reverse: it reports cached tokens on the second
|
||||
call and no cache writes at all. gpt-5.3-codex is Responses-only, which is what
|
||||
forces the /v1/messages bridge, and it starts reporting cache reads once the
|
||||
prefix is a few thousand tokens rather than one.
|
||||
|
||||
OpenAI caching is best-effort, so each test retries with a fresh prefix (new
|
||||
marker = brand-new cache identity) up to three times before failing; the prime and
|
||||
measured calls share the prefix but differ in the trailing question, which defeats
|
||||
the proxy's own response cache without touching the provider's prefix cache.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from cost_rows import (
|
||||
CostRow,
|
||||
approx_equal,
|
||||
assert_fresh_tokens_billed_at,
|
||||
assert_total_is_sum_of_components,
|
||||
cacheable_prefix,
|
||||
poll_cost_row,
|
||||
poll_cost_row_where,
|
||||
register_priced_model,
|
||||
)
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import AnthropicMessagesBody, ChatBody, ChatMessage, LiteLLMParamsBody
|
||||
from pydantic import BaseModel
|
||||
from spend_e2e_client import SpendClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
CACHE_WRITE_BACKEND = "openai/gpt-5.6-luna"
|
||||
CACHE_READ_BACKEND = "openai/gpt-5.5"
|
||||
BRIDGE_BACKEND = "openai/gpt-5.3-codex"
|
||||
BRIDGE_PREFIX_WORDS = 3000
|
||||
OPENAI_API_KEY = "os.environ/OPENAI_API_KEY"
|
||||
CACHE_ATTEMPTS = 3
|
||||
|
||||
INPUT_RATE = 4e-05
|
||||
OUTPUT_RATE = 8e-05
|
||||
CACHE_READ_RATE = 1e-05
|
||||
CACHE_WRITE_RATE = 5e-05
|
||||
|
||||
PRIME_QUESTION = "Reply with the single word ready."
|
||||
REASONING_QUESTION = "Compute 47*83 - 19*7 step by step, then reply with just the final number."
|
||||
|
||||
|
||||
class _StreamChunk(BaseModel):
|
||||
id: str | None = None
|
||||
|
||||
|
||||
def _cache_priced_params(backend: str) -> LiteLLMParamsBody:
|
||||
return LiteLLMParamsBody(
|
||||
model=backend,
|
||||
api_key=OPENAI_API_KEY,
|
||||
input_cost_per_token=INPUT_RATE,
|
||||
output_cost_per_token=OUTPUT_RATE,
|
||||
cache_read_input_token_cost=CACHE_READ_RATE,
|
||||
cache_creation_input_token_cost=CACHE_WRITE_RATE,
|
||||
)
|
||||
|
||||
|
||||
def _chat_body(model: str, content: str, *, stream: bool = False) -> ChatBody:
|
||||
return ChatBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content=content)],
|
||||
stream=stream,
|
||||
max_completion_tokens=4000,
|
||||
)
|
||||
|
||||
|
||||
def _require_row(client: SpendClient, request_id: str) -> CostRow:
|
||||
row = poll_cost_row(client.proxy, request_id)
|
||||
assert row is not None, f"no spend row with a cost breakdown landed for {request_id}"
|
||||
return row
|
||||
|
||||
|
||||
def _assert_cache_read_billed(row: CostRow) -> None:
|
||||
assert row.breakdown.cache_read_cost is not None and approx_equal(
|
||||
row.breakdown.cache_read_cost, row.cache_read_tokens * CACHE_READ_RATE
|
||||
), (
|
||||
f"cache_read_cost {row.breakdown.cache_read_cost} != "
|
||||
f"{row.cache_read_tokens} cached tokens * {CACHE_READ_RATE}"
|
||||
)
|
||||
assert_fresh_tokens_billed_at(row, INPUT_RATE)
|
||||
assert_total_is_sum_of_components(row)
|
||||
|
||||
|
||||
class TestCacheCostAccounting:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cache_write.bills_cache_creation_rate")
|
||||
def test_cache_write_tokens_billed_at_cache_creation_rate(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model = register_priced_model(
|
||||
client.proxy, resources, "cache-write-priced", _cache_priced_params(CACHE_WRITE_BACKEND)
|
||||
)
|
||||
|
||||
for _ in range(CACHE_ATTEMPTS):
|
||||
prompt = f"{cacheable_prefix(unique_marker())}\n{PRIME_QUESTION}"
|
||||
chat = unwrap(client.proxy.chat(scoped_key, _chat_body(model, prompt)))
|
||||
assert chat.id, f"chat response carried no id: {chat}"
|
||||
row = _require_row(client, chat.id)
|
||||
if row.cache_creation_tokens > 0:
|
||||
break
|
||||
else:
|
||||
pytest.fail(
|
||||
f"OpenAI reported no cache-write tokens across {CACHE_ATTEMPTS} fresh "
|
||||
"~2k-token prompts; the cache-write billing path was never exercised"
|
||||
)
|
||||
|
||||
assert row.breakdown.cache_creation_cost is not None and approx_equal(
|
||||
row.breakdown.cache_creation_cost, row.cache_creation_tokens * CACHE_WRITE_RATE
|
||||
), (
|
||||
f"cache_creation_cost {row.breakdown.cache_creation_cost} != "
|
||||
f"{row.cache_creation_tokens} cache-write tokens * {CACHE_WRITE_RATE}"
|
||||
)
|
||||
assert_fresh_tokens_billed_at(row, INPUT_RATE)
|
||||
assert_total_is_sum_of_components(row)
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cost_breakdown.reports_component_costs")
|
||||
def test_cost_breakdown_reports_component_costs(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model = register_priced_model(
|
||||
client.proxy, resources, "breakdown-priced", _cache_priced_params(CACHE_READ_BACKEND)
|
||||
)
|
||||
|
||||
for _ in range(CACHE_ATTEMPTS):
|
||||
prefix = cacheable_prefix(unique_marker())
|
||||
unwrap(client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{PRIME_QUESTION}")))
|
||||
chat = unwrap(
|
||||
client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{REASONING_QUESTION}"))
|
||||
)
|
||||
assert chat.id, f"chat response carried no id: {chat}"
|
||||
row = _require_row(client, chat.id)
|
||||
if row.cache_read_tokens > 0:
|
||||
break
|
||||
else:
|
||||
pytest.fail(
|
||||
f"no cache read landed across {CACHE_ATTEMPTS} prime+read rounds; "
|
||||
"the component-cost breakdown was never exercised with cached input"
|
||||
)
|
||||
|
||||
usage = chat.usage
|
||||
assert usage is not None and usage.completion_tokens_details is not None, (
|
||||
f"no completion token details on the measured call: {chat}"
|
||||
)
|
||||
reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0
|
||||
assert reasoning_tokens > 0, f"the reasoning question produced no reasoning tokens: {usage}"
|
||||
|
||||
breakdown = row.breakdown
|
||||
assert breakdown.output_cost is not None and approx_equal(
|
||||
breakdown.output_cost, (row.completion_tokens or 0) * OUTPUT_RATE
|
||||
), (
|
||||
f"output_cost {breakdown.output_cost} != "
|
||||
f"{row.completion_tokens} completion tokens * {OUTPUT_RATE}"
|
||||
)
|
||||
assert breakdown.reasoning_cost is not None and approx_equal(
|
||||
breakdown.reasoning_cost, reasoning_tokens * OUTPUT_RATE
|
||||
), (
|
||||
f"reasoning_cost {breakdown.reasoning_cost} != "
|
||||
f"{reasoning_tokens} reasoning tokens * {OUTPUT_RATE}"
|
||||
)
|
||||
assert breakdown.reasoning_cost <= (breakdown.output_cost or 0.0) * 1.01, (
|
||||
f"reasoning_cost {breakdown.reasoning_cost} exceeds output_cost "
|
||||
f"{breakdown.output_cost}; reasoning must be a subset of output"
|
||||
)
|
||||
_assert_cache_read_billed(row)
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.stream_cache_read.bills_cache_read_rate")
|
||||
def test_streaming_cache_read_billed_at_cache_read_rate(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model = register_priced_model(
|
||||
client.proxy, resources, "stream-cache-priced", _cache_priced_params(CACHE_READ_BACKEND)
|
||||
)
|
||||
|
||||
for _ in range(CACHE_ATTEMPTS):
|
||||
prefix = cacheable_prefix(unique_marker())
|
||||
unwrap(client.proxy.chat(scoped_key, _chat_body(model, f"{prefix}\n{PRIME_QUESTION}")))
|
||||
result = client.proxy.chat_stream(
|
||||
scoped_key,
|
||||
_chat_body(model, f"{prefix}\nReply with the single word cached.", stream=True),
|
||||
)
|
||||
assert result.ok and result.stream_events, (
|
||||
f"streamed chat failed (status {result.status_code}): {result.body[:300]}"
|
||||
)
|
||||
stream_id = _StreamChunk.model_validate_json(result.stream_events[0]).id
|
||||
assert stream_id, f"first stream chunk carried no id: {result.stream_events[0][:200]}"
|
||||
row = _require_row(client, stream_id)
|
||||
if row.cache_read_tokens > 0:
|
||||
break
|
||||
else:
|
||||
pytest.fail(
|
||||
f"no cache read landed across {CACHE_ATTEMPTS} prime+stream rounds; "
|
||||
"streaming cache-read billing was never exercised"
|
||||
)
|
||||
|
||||
_assert_cache_read_billed(row)
|
||||
|
||||
@pytest.mark.covers("quota_management.spend_tracking.messages_bridge.keeps_cache_tokens")
|
||||
def test_messages_bridge_keeps_cache_tokens(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model = register_priced_model(
|
||||
client.proxy, resources, "bridge-cache-priced", _cache_priced_params(BRIDGE_BACKEND)
|
||||
)
|
||||
|
||||
def bridge_call(content: str) -> int:
|
||||
response = unwrap(
|
||||
client.proxy.messages(
|
||||
scoped_key,
|
||||
AnthropicMessagesBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content=content)],
|
||||
max_tokens=4000,
|
||||
),
|
||||
)
|
||||
)
|
||||
assert response.usage is not None, f"bridged response carried no usage: {response}"
|
||||
return response.usage.cache_read_input_tokens or 0
|
||||
|
||||
for _ in range(CACHE_ATTEMPTS):
|
||||
prefix = cacheable_prefix(unique_marker(), words=BRIDGE_PREFIX_WORDS)
|
||||
bridge_call(f"{prefix}\n{PRIME_QUESTION}")
|
||||
if bridge_call(f"{prefix}\nReply with the single word bridged.") > 0:
|
||||
break
|
||||
else:
|
||||
pytest.fail(
|
||||
f"no cache read survived {CACHE_ATTEMPTS} bridged prime+read rounds; "
|
||||
"cache tokens are not surviving the anthropic-messages -> Responses bridge"
|
||||
)
|
||||
|
||||
row = poll_cost_row_where(client.proxy, scoped_key, lambda r: r.cache_read_tokens > 0)
|
||||
assert row is not None, (
|
||||
"the bridged call reported cached tokens but no spend row for the key "
|
||||
"recorded any; the cache tokens were dropped on the way to the bill"
|
||||
)
|
||||
_assert_cache_read_billed(row)
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
"""Live e2e: the per-component x-litellm-response-cost-* headers keep their contract.
|
||||
|
||||
Pins the header contract shipped in #36965: alongside the x-litellm-response-cost
|
||||
total, every response carries the component costs (input, output, cache-read,
|
||||
cache-creation, reasoning, tool-usage), where input covers only fresh tokens (the
|
||||
cache components are subtracted out) so the components sum to the total, and
|
||||
reasoning stays a subset of output.
|
||||
|
||||
The deployment carries distinct custom rates per component, a prime call fills the
|
||||
provider's prefix cache, and the measured call re-reads it, so the cache-read
|
||||
header is exercised with a real nonzero value instead of passing vacuously. The
|
||||
backend is gpt-5.5 because it reports cached tokens on the second call; the
|
||||
gpt-5.6 line reports cache writes and never a read, which would leave the
|
||||
cache-read header at zero forever. The raw-transport send is used because the
|
||||
typed chat client validates bodies and drops headers. OpenAI caching is
|
||||
best-effort, so the prime+measure round retries with a fresh prefix before
|
||||
failing.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from cost_rows import approx_equal, cacheable_prefix, register_priced_model
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import StreamingResponse
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody
|
||||
from spend_e2e_client import SpendClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND = "openai/gpt-5.5"
|
||||
OPENAI_API_KEY = "os.environ/OPENAI_API_KEY"
|
||||
CACHE_ATTEMPTS = 3
|
||||
|
||||
INPUT_RATE = 4e-05
|
||||
OUTPUT_RATE = 8e-05
|
||||
CACHE_READ_RATE = 1e-05
|
||||
CACHE_WRITE_RATE = 5e-05
|
||||
|
||||
COMPONENT_HEADERS = (
|
||||
"x-litellm-response-cost-input",
|
||||
"x-litellm-response-cost-cache-read",
|
||||
"x-litellm-response-cost-cache-creation",
|
||||
"x-litellm-response-cost-output",
|
||||
"x-litellm-response-cost-tool-usage",
|
||||
)
|
||||
|
||||
|
||||
def _header_cost(response: StreamingResponse, name: str) -> float:
|
||||
value = response.headers.get(name)
|
||||
return float(value) if value not in (None, "", "None") else 0.0
|
||||
|
||||
|
||||
class TestCostHeaders:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.cost_headers.additive_components")
|
||||
def test_component_cost_headers_sum_to_total(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model = register_priced_model(
|
||||
client.proxy,
|
||||
resources,
|
||||
"header-priced",
|
||||
LiteLLMParamsBody(
|
||||
model=BACKEND,
|
||||
api_key=OPENAI_API_KEY,
|
||||
input_cost_per_token=INPUT_RATE,
|
||||
output_cost_per_token=OUTPUT_RATE,
|
||||
cache_read_input_token_cost=CACHE_READ_RATE,
|
||||
cache_creation_input_token_cost=CACHE_WRITE_RATE,
|
||||
),
|
||||
)
|
||||
|
||||
def priced_call(content: str) -> StreamingResponse:
|
||||
response = client.proxy.transport.send(
|
||||
"/chat/completions",
|
||||
headers=client.proxy.transport.bearer(scoped_key),
|
||||
json=ChatBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content=content)],
|
||||
max_completion_tokens=4000,
|
||||
),
|
||||
)
|
||||
assert response.ok, f"chat failed (status {response.status_code}): {response.body[:300]}"
|
||||
return response
|
||||
|
||||
for _ in range(CACHE_ATTEMPTS):
|
||||
prefix = cacheable_prefix(unique_marker())
|
||||
priced_call(f"{prefix}\nReply with the single word ready.")
|
||||
measured = priced_call(f"{prefix}\nReply with the single word measured.")
|
||||
if _header_cost(measured, "x-litellm-response-cost-cache-read") > 0:
|
||||
break
|
||||
else:
|
||||
pytest.fail(
|
||||
f"no cache read landed across {CACHE_ATTEMPTS} prime+measure rounds; "
|
||||
"the cache-read cost header was never exercised with a nonzero value"
|
||||
)
|
||||
|
||||
total = measured.response_cost
|
||||
assert total is not None and total > 0, (
|
||||
f"x-litellm-response-cost missing or zero: {measured.headers}"
|
||||
)
|
||||
component_sum = sum(_header_cost(measured, name) for name in COMPONENT_HEADERS)
|
||||
assert approx_equal(component_sum, total), (
|
||||
f"component headers sum to {component_sum}, not the total {total}: "
|
||||
f"{ {name: measured.headers.get(name) for name in COMPONENT_HEADERS} }"
|
||||
)
|
||||
|
||||
reasoning = _header_cost(measured, "x-litellm-response-cost-reasoning")
|
||||
output = _header_cost(measured, "x-litellm-response-cost-output")
|
||||
assert reasoning <= output * 1.01, (
|
||||
f"reasoning header {reasoning} exceeds output header {output}; "
|
||||
"reasoning must be a subset of output"
|
||||
)
|
||||
|
||||
usage = ChatResponse.model_validate_json(measured.body).usage
|
||||
assert usage is not None, f"measured response carried no usage: {measured.body[:300]}"
|
||||
cached_tokens = (
|
||||
usage.prompt_tokens_details.cached_tokens or 0 if usage.prompt_tokens_details else 0
|
||||
)
|
||||
cache_creation_tokens = usage.cache_creation_input_tokens or 0
|
||||
assert cached_tokens > 0, f"cache-read header nonzero but usage shows no cached tokens: {usage}"
|
||||
assert approx_equal(
|
||||
_header_cost(measured, "x-litellm-response-cost-cache-read"),
|
||||
cached_tokens * CACHE_READ_RATE,
|
||||
), (
|
||||
f"cache-read header {measured.headers.get('x-litellm-response-cost-cache-read')} != "
|
||||
f"{cached_tokens} cached tokens * {CACHE_READ_RATE}"
|
||||
)
|
||||
fresh_tokens = (usage.prompt_tokens or 0) - cached_tokens - cache_creation_tokens
|
||||
assert approx_equal(
|
||||
_header_cost(measured, "x-litellm-response-cost-input"), fresh_tokens * INPUT_RATE
|
||||
), (
|
||||
f"input header {measured.headers.get('x-litellm-response-cost-input')} != "
|
||||
f"{fresh_tokens} fresh tokens * {INPUT_RATE}; the input component is not "
|
||||
"subtracting the cache components"
|
||||
)
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
"""Live e2e: the /openai passthrough injects usage.cost into streaming usage frames.
|
||||
|
||||
Pins #36503: with the proxy running `include_cost_in_streaming_usage: true`, a
|
||||
streamed call through the provider passthrough surface must carry the computed
|
||||
cost inside the final usage-only SSE frame, the same contract the native
|
||||
/chat/completions stream has. Providers never send `cost` themselves, so a
|
||||
nonzero value proves the proxy computed and injected it on the passthrough path.
|
||||
|
||||
The row-side spend accounting for passthrough calls is covered elsewhere; this
|
||||
test pins only the in-stream cost surface, which clients read without ever
|
||||
touching /spend/logs.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from models import ChatBody, ChatMessage, StreamOptions, Usage
|
||||
from spend_e2e_client import SpendClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
OPENAI_MODEL = "gpt-5.6-luna"
|
||||
|
||||
|
||||
class _StreamFrame(BaseModel):
|
||||
usage: Usage | None = None
|
||||
|
||||
|
||||
class TestPassthroughStreamCost:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.passthrough_stream.injects_usage_cost")
|
||||
def test_passthrough_stream_final_usage_frame_carries_cost(
|
||||
self, client: SpendClient, scoped_key: str
|
||||
) -> None:
|
||||
result = client.proxy.transport.send(
|
||||
"/openai/v1/chat/completions",
|
||||
headers=client.proxy.transport.bearer(scoped_key),
|
||||
json=ChatBody(
|
||||
model=OPENAI_MODEL,
|
||||
messages=[
|
||||
ChatMessage(
|
||||
role="user",
|
||||
content=f"{unique_marker()} Reply with the single word passthrough.",
|
||||
)
|
||||
],
|
||||
stream=True,
|
||||
stream_options=StreamOptions(),
|
||||
),
|
||||
stream=True,
|
||||
)
|
||||
assert result.ok and result.stream_events, (
|
||||
f"passthrough stream failed (status {result.status_code}): {result.body[:300]}"
|
||||
)
|
||||
|
||||
usage_frames = [
|
||||
frame.usage
|
||||
for frame in (_StreamFrame.model_validate_json(event) for event in result.stream_events)
|
||||
if frame.usage is not None
|
||||
]
|
||||
assert usage_frames, (
|
||||
f"no usage frame in the passthrough stream despite stream_options.include_usage; "
|
||||
f"last event: {result.stream_events[-1][:300]}"
|
||||
)
|
||||
|
||||
final_usage = usage_frames[-1]
|
||||
assert final_usage.cost is not None and final_usage.cost > 0, (
|
||||
f"final passthrough usage frame carries no injected cost: {final_usage}"
|
||||
)
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
"""Live e2e: a service_tier request bills every component at the tier's own rates.
|
||||
|
||||
Pins the tier-billing fixes (#35923, #35925): a priority-tier call must price
|
||||
input and output at the deployment's `*_priority` rates, including the reasoning
|
||||
tokens inside output (the shipped bug billed reasoning at the default-tier rate),
|
||||
and the spend row must record the tier the bill was computed on.
|
||||
|
||||
The deployment carries custom base AND priority rates, each distinct, so a bill
|
||||
computed from the wrong tier (or a mix) cannot match the expected numbers. The
|
||||
prompt is a fresh unique marker per run, keeping cached tokens out of the math.
|
||||
The response's own `service_tier` echo is asserted first: if OpenAI ever declined
|
||||
priority processing and served the default tier, the test fails there instead of
|
||||
producing a vacuous rate comparison.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from cost_rows import (
|
||||
approx_equal,
|
||||
assert_fresh_tokens_billed_at,
|
||||
assert_total_is_sum_of_components,
|
||||
poll_cost_row,
|
||||
register_priced_model,
|
||||
)
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import unwrap
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, LiteLLMParamsBody
|
||||
from spend_e2e_client import SpendClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
BACKEND = "openai/gpt-5.6-luna"
|
||||
OPENAI_API_KEY = "os.environ/OPENAI_API_KEY"
|
||||
|
||||
INPUT_RATE = 4e-05
|
||||
OUTPUT_RATE = 8e-05
|
||||
PRIORITY_INPUT_RATE = 6e-05
|
||||
PRIORITY_OUTPUT_RATE = 1.6e-04
|
||||
|
||||
|
||||
class TestServiceTierPricing:
|
||||
@pytest.mark.covers("quota_management.spend_tracking.service_tier.bills_tier_rates")
|
||||
def test_priority_tier_bills_priority_rates(
|
||||
self, client: SpendClient, resources: ResourceManager, scoped_key: str
|
||||
) -> None:
|
||||
model = register_priced_model(
|
||||
client.proxy,
|
||||
resources,
|
||||
"tier-priced",
|
||||
LiteLLMParamsBody(
|
||||
model=BACKEND,
|
||||
api_key=OPENAI_API_KEY,
|
||||
input_cost_per_token=INPUT_RATE,
|
||||
output_cost_per_token=OUTPUT_RATE,
|
||||
input_cost_per_token_priority=PRIORITY_INPUT_RATE,
|
||||
output_cost_per_token_priority=PRIORITY_OUTPUT_RATE,
|
||||
),
|
||||
)
|
||||
|
||||
chat = unwrap(
|
||||
client.proxy.chat(
|
||||
scoped_key,
|
||||
ChatBody(
|
||||
model=model,
|
||||
messages=[
|
||||
ChatMessage(
|
||||
role="user",
|
||||
content=(
|
||||
f"{unique_marker()} Compute 47*83 - 19*7 step by step, "
|
||||
"then reply with just the final number."
|
||||
),
|
||||
)
|
||||
],
|
||||
max_completion_tokens=4000,
|
||||
service_tier="priority",
|
||||
),
|
||||
)
|
||||
)
|
||||
assert chat.service_tier == "priority", (
|
||||
f"OpenAI served tier {chat.service_tier!r} instead of priority; "
|
||||
"tier billing was never exercised"
|
||||
)
|
||||
assert chat.id, f"chat response carried no id: {chat}"
|
||||
|
||||
row = poll_cost_row(client.proxy, chat.id)
|
||||
assert row is not None, f"no spend row with a cost breakdown landed for {chat.id}"
|
||||
breakdown = row.breakdown
|
||||
|
||||
assert breakdown.service_tier == "priority", (
|
||||
f"the bill records pricing basis {breakdown.service_tier!r}, not priority"
|
||||
)
|
||||
|
||||
assert_fresh_tokens_billed_at(row, PRIORITY_INPUT_RATE)
|
||||
assert breakdown.output_cost is not None and approx_equal(
|
||||
breakdown.output_cost, (row.completion_tokens or 0) * PRIORITY_OUTPUT_RATE
|
||||
), (
|
||||
f"output_cost {breakdown.output_cost} != {row.completion_tokens} tokens * priority rate "
|
||||
f"{PRIORITY_OUTPUT_RATE} (base rate would give {(row.completion_tokens or 0) * OUTPUT_RATE})"
|
||||
)
|
||||
|
||||
usage = chat.usage
|
||||
assert usage is not None and usage.completion_tokens_details is not None, (
|
||||
f"no completion token details on the priority call: {chat}"
|
||||
)
|
||||
reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0
|
||||
assert reasoning_tokens > 0, f"the reasoning question produced no reasoning tokens: {usage}"
|
||||
assert breakdown.reasoning_cost is not None and approx_equal(
|
||||
breakdown.reasoning_cost, reasoning_tokens * PRIORITY_OUTPUT_RATE
|
||||
), (
|
||||
f"reasoning_cost {breakdown.reasoning_cost} != {reasoning_tokens} reasoning tokens * "
|
||||
f"priority rate {PRIORITY_OUTPUT_RATE} (the default-tier rate would give "
|
||||
f"{reasoning_tokens * OUTPUT_RATE})"
|
||||
)
|
||||
|
||||
assert_total_is_sum_of_components(row)
|
||||
Loading…
Add table
Reference in a new issue