mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(spend): price auto-router savings through litellm's cost engine and surface them
Three defects, one cause: the driver was wired by hand at each stage of the savings pipeline instead of going through the owner of each stage. Pricing re-derived per-token arithmetic instead of calling the cost engine. `prompt_tokens` already includes cache-read and cache-creation tokens, so charging the whole total at the flat input rate and then subtracting a separate cache-write penalty priced those tokens twice. Both arms now price the identical usage through `generic_cost_per_token`, so each token is charged once in its own dimension and tiered rates, ephemeral cache-write tiers and regional uplifts stay consistent with what the request was actually billed. On a cache-heavy opus-to-haiku switch the old formula reported $0.0458 against a true $0.0717. The savings baseline was recorded by a second per-attempt writer sitting beside the routing decision, and the exit path that runs when no pre-routing strategy applies only cleared the decision. A fallback to a plain model group therefore kept the previous attempt's baseline, letting a caller who forces a router failure inflate the recorded savings. Both facts now travel from one response through one recorder, so no exit can clear one and leave the other. Aggregation summed every daily metric except this one, so two requests sharing a rollup key kept only the first value, and since the cross-pod Redis drain runs the same merge the field was dropped on every flush. The driver was also absent from the entire read path: no column in the rollup query, no accumulation, no field on the response model. The dashboard read a key the API never sent, so the card, donut segment and graph series would have rendered $0.00 forever however much routing saved. Wiring the write path without the read path is the failure this had already shipped, so the drivers are now enumerated from the response model itself and each is asserted to be summed, accumulated, carried and totalled. Also collapses the twelve hand-written per-field blocks in the daily upsert into one enumeration feeding both the create and the increment.
This commit is contained in:
Tin Chi Lo
parent
e30b7f2f1f
commit
4897f6ec88
15 changed files with 430 additions and 237 deletions
|
|
@ -4560,6 +4560,7 @@ class BaseDailySpendTransaction(TypedDict):
|
|||
# cost-savings metrics (dollars, priced per request before aggregation)
|
||||
compression_savings_spend: float
|
||||
prompt_caching_savings_spend: float
|
||||
autorouter_savings_spend: float
|
||||
|
||||
# request level metrics
|
||||
spend: float
|
||||
|
|
|
|||
|
|
@ -1556,6 +1556,30 @@ class DBSpendUpdateWriter:
|
|||
# Get the table dynamically
|
||||
table = getattr(batcher, table_name)
|
||||
|
||||
# Additive metrics that older queued rows may omit; one
|
||||
# enumeration feeds both the create and the increment below
|
||||
optional_metrics = {
|
||||
field: value
|
||||
for field, value in (
|
||||
("cache_read_input_tokens", transaction.get("cache_read_input_tokens")),
|
||||
(
|
||||
"cache_creation_input_tokens",
|
||||
transaction.get("cache_creation_input_tokens"),
|
||||
),
|
||||
("compression_saved_tokens", transaction.get("compression_saved_tokens")),
|
||||
(
|
||||
"compression_savings_spend",
|
||||
transaction.get("compression_savings_spend"),
|
||||
),
|
||||
(
|
||||
"prompt_caching_savings_spend",
|
||||
transaction.get("prompt_caching_savings_spend"),
|
||||
),
|
||||
("autorouter_savings_spend", transaction.get("autorouter_savings_spend")),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
|
||||
# Common data structure for both create and update
|
||||
common_data = {
|
||||
entity_id_field: entity_id,
|
||||
|
|
@ -1572,34 +1596,9 @@ class DBSpendUpdateWriter:
|
|||
"api_requests": transaction["api_requests"],
|
||||
"successful_requests": transaction["successful_requests"],
|
||||
"failed_requests": transaction["failed_requests"],
|
||||
**optional_metrics,
|
||||
}
|
||||
|
||||
# Add cache-related fields if they exist
|
||||
if "cache_read_input_tokens" in transaction:
|
||||
common_data["cache_read_input_tokens"] = transaction.get(
|
||||
"cache_read_input_tokens", 0
|
||||
)
|
||||
if "cache_creation_input_tokens" in transaction:
|
||||
common_data["cache_creation_input_tokens"] = transaction.get(
|
||||
"cache_creation_input_tokens", 0
|
||||
)
|
||||
if "compression_saved_tokens" in transaction:
|
||||
common_data["compression_saved_tokens"] = transaction.get(
|
||||
"compression_saved_tokens", 0
|
||||
)
|
||||
if "compression_savings_spend" in transaction:
|
||||
common_data["compression_savings_spend"] = transaction.get(
|
||||
"compression_savings_spend", 0
|
||||
)
|
||||
if "prompt_caching_savings_spend" in transaction:
|
||||
common_data["prompt_caching_savings_spend"] = transaction.get(
|
||||
"prompt_caching_savings_spend", 0
|
||||
)
|
||||
if "autorouter_savings_spend" in transaction:
|
||||
common_data["autorouter_savings_spend"] = transaction.get(
|
||||
"autorouter_savings_spend", 0
|
||||
)
|
||||
|
||||
if entity_type == "tag" and "request_id" in transaction:
|
||||
common_data["request_id"] = transaction.get("request_id")
|
||||
|
||||
|
|
@ -1611,36 +1610,9 @@ class DBSpendUpdateWriter:
|
|||
"api_requests": {"increment": transaction["api_requests"]},
|
||||
"successful_requests": {"increment": transaction["successful_requests"]},
|
||||
"failed_requests": {"increment": transaction["failed_requests"]},
|
||||
**{field: {"increment": value} for field, value in optional_metrics.items()},
|
||||
}
|
||||
|
||||
# Add cache-related fields to update if they exist
|
||||
if "cache_read_input_tokens" in transaction:
|
||||
update_data["cache_read_input_tokens"] = {
|
||||
"increment": transaction.get("cache_read_input_tokens", 0)
|
||||
}
|
||||
if "cache_creation_input_tokens" in transaction:
|
||||
update_data["cache_creation_input_tokens"] = {
|
||||
"increment": transaction.get("cache_creation_input_tokens", 0)
|
||||
}
|
||||
if "compression_saved_tokens" in transaction:
|
||||
update_data["compression_saved_tokens"] = {
|
||||
"increment": transaction.get("compression_saved_tokens", 0)
|
||||
}
|
||||
if "compression_savings_spend" in transaction:
|
||||
update_data["compression_savings_spend"] = {
|
||||
"increment": transaction.get("compression_savings_spend", 0)
|
||||
}
|
||||
if "prompt_caching_savings_spend" in transaction:
|
||||
update_data["prompt_caching_savings_spend"] = {
|
||||
"increment": transaction.get("prompt_caching_savings_spend", 0)
|
||||
}
|
||||
if "autorouter_savings_spend" in transaction:
|
||||
update_data[
|
||||
"autorouter_savings_spend"
|
||||
] = { # mutable-ok: extends preset dict following compression_savings_spend pattern
|
||||
"increment": transaction.get("autorouter_savings_spend", 0)
|
||||
}
|
||||
|
||||
if entity_type == "tag" and "request_id" in transaction:
|
||||
update_data["request_id"] = transaction.get("request_id")
|
||||
|
||||
|
|
@ -1891,17 +1863,13 @@ class DBSpendUpdateWriter:
|
|||
|
||||
cache_read_input_tokens = _extract_cache_read_tokens(usage_obj)
|
||||
compression_saved_tokens = extract_compression_saved_tokens(_metadata)
|
||||
cache_creation_input_tokens = _extract_cache_creation_tokens(usage_obj)
|
||||
|
||||
savings_spend = compute_savings_spend(
|
||||
model=payload.get("model", None),
|
||||
custom_llm_provider=payload.get("custom_llm_provider", None),
|
||||
compression_saved_tokens=compression_saved_tokens,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
baseline_model=_metadata.get("auto_router_savings_baseline_model"),
|
||||
prompt_tokens=payload.get("prompt_tokens", 0),
|
||||
completion_tokens=payload.get("completion_tokens", 0),
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
usage_object=usage_obj,
|
||||
)
|
||||
|
||||
daily_transaction = BaseDailySpendTransaction(
|
||||
|
|
@ -1919,7 +1887,7 @@ class DBSpendUpdateWriter:
|
|||
successful_requests=1 if request_status == "success" else 0,
|
||||
failed_requests=1 if request_status != "success" else 0,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
cache_creation_input_tokens=_extract_cache_creation_tokens(usage_obj),
|
||||
compression_saved_tokens=compression_saved_tokens,
|
||||
compression_savings_spend=savings_spend.compression,
|
||||
prompt_caching_savings_spend=savings_spend.prompt_caching,
|
||||
|
|
|
|||
|
|
@ -134,6 +134,10 @@ class DailySpendUpdateQueue(BaseUpdateQueue):
|
|||
payload.get("prompt_caching_savings_spend", 0) or 0
|
||||
) + daily_transaction.get("prompt_caching_savings_spend", 0)
|
||||
|
||||
daily_transaction["autorouter_savings_spend"] = (
|
||||
payload.get("autorouter_savings_spend", 0) or 0
|
||||
) + daily_transaction.get("autorouter_savings_spend", 0)
|
||||
|
||||
else:
|
||||
aggregated_daily_spend_update_transactions[_key] = deepcopy(payload)
|
||||
return aggregated_daily_spend_update_transactions
|
||||
|
|
|
|||
|
|
@ -95,6 +95,9 @@ class DailySpendRecord(Protocol):
|
|||
@property
|
||||
def prompt_caching_savings_spend(self) -> float: ...
|
||||
|
||||
@property
|
||||
def autorouter_savings_spend(self) -> float: ...
|
||||
|
||||
@property
|
||||
def api_requests(self) -> int: ...
|
||||
|
||||
|
|
@ -135,6 +138,7 @@ class _GroupingSetsRow(SimpleNamespace):
|
|||
compression_saved_tokens: int | None
|
||||
compression_savings_spend: float | None
|
||||
prompt_caching_savings_spend: float | None
|
||||
autorouter_savings_spend: float | None
|
||||
api_requests: int | None
|
||||
successful_requests: int | None
|
||||
failed_requests: int | None
|
||||
|
|
@ -158,6 +162,7 @@ def update_metrics(existing_metrics: SpendMetrics, record: DailySpendRecord) ->
|
|||
existing_metrics.compression_saved_tokens += record.compression_saved_tokens or 0
|
||||
existing_metrics.compression_savings_spend += record.compression_savings_spend or 0
|
||||
existing_metrics.prompt_caching_savings_spend += record.prompt_caching_savings_spend or 0
|
||||
existing_metrics.autorouter_savings_spend += record.autorouter_savings_spend or 0
|
||||
existing_metrics.api_requests += record.api_requests or 0
|
||||
existing_metrics.successful_requests += record.successful_requests or 0
|
||||
existing_metrics.failed_requests += record.failed_requests or 0
|
||||
|
|
@ -590,6 +595,7 @@ def _build_aggregated_sql_query(
|
|||
SUM(compression_saved_tokens)::bigint AS compression_saved_tokens,
|
||||
SUM(compression_savings_spend)::float AS compression_savings_spend,
|
||||
SUM(prompt_caching_savings_spend)::float AS prompt_caching_savings_spend,
|
||||
SUM(autorouter_savings_spend)::float AS autorouter_savings_spend,
|
||||
SUM(api_requests)::bigint AS api_requests,
|
||||
SUM(successful_requests)::bigint AS successful_requests,
|
||||
SUM(failed_requests)::bigint AS failed_requests
|
||||
|
|
@ -732,6 +738,7 @@ def _record_to_spend_metrics(record: _GroupingSetsRow) -> SpendMetrics:
|
|||
compression_saved_tokens=record.compression_saved_tokens or 0,
|
||||
compression_savings_spend=record.compression_savings_spend or 0,
|
||||
prompt_caching_savings_spend=record.prompt_caching_savings_spend or 0,
|
||||
autorouter_savings_spend=record.autorouter_savings_spend or 0,
|
||||
api_requests=record.api_requests or 0,
|
||||
successful_requests=record.successful_requests or 0,
|
||||
failed_requests=record.failed_requests or 0,
|
||||
|
|
@ -986,6 +993,7 @@ async def get_daily_activity(
|
|||
total_compression_saved_tokens=metadata_metrics.compression_saved_tokens,
|
||||
total_compression_savings_spend=metadata_metrics.compression_savings_spend,
|
||||
total_prompt_caching_savings_spend=metadata_metrics.prompt_caching_savings_spend,
|
||||
total_autorouter_savings_spend=metadata_metrics.autorouter_savings_spend,
|
||||
page=page,
|
||||
total_pages=-(-total_count // page_size), # Ceiling division
|
||||
has_more=(page * page_size) < total_count,
|
||||
|
|
@ -1075,6 +1083,7 @@ async def get_daily_activity_aggregated(
|
|||
total_compression_saved_tokens=aggregated["totals"].compression_saved_tokens,
|
||||
total_compression_savings_spend=aggregated["totals"].compression_savings_spend,
|
||||
total_prompt_caching_savings_spend=aggregated["totals"].prompt_caching_savings_spend,
|
||||
total_autorouter_savings_spend=aggregated["totals"].autorouter_savings_spend,
|
||||
page=1,
|
||||
total_pages=1,
|
||||
has_more=False,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ from typing import NamedTuple
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
class SavingsSpend(NamedTuple):
|
||||
|
|
@ -20,42 +22,51 @@ class SavingsSpend(NamedTuple):
|
|||
autorouter: float = 0.0
|
||||
|
||||
|
||||
class _ModelRates(NamedTuple):
|
||||
input: float
|
||||
output: float
|
||||
cache_read: float
|
||||
cache_write: float
|
||||
|
||||
|
||||
def _model_rates(model: str | None, custom_llm_provider: str | None) -> _ModelRates:
|
||||
def _input_and_cache_read_cost(model: str | None, custom_llm_provider: str | None) -> tuple[float, float]:
|
||||
"""
|
||||
Per-token prices for a model, in its four billed dimensions.
|
||||
Return ``(input_cost_per_token, cache_read_cost_per_token)`` for a model.
|
||||
|
||||
Falls open to all-zero rates when the model is unknown so savings degrade to
|
||||
zero rather than raising inside the spend writer. When a model has no separate
|
||||
cache-read price the cache-read rate mirrors the input rate, which yields zero
|
||||
caching savings; a missing cache-write price falls back to the input rate,
|
||||
which is what providers without a distinct cache-creation charge bill.
|
||||
Falls open to ``(0.0, 0.0)`` when the model is unknown so savings degrade to
|
||||
zero rather than raising inside the spend writer. When a model has no
|
||||
separate cache-read price the cache-read cost mirrors the input cost, which
|
||||
yields zero caching savings.
|
||||
"""
|
||||
if not model:
|
||||
return _ModelRates(0.0, 0.0, 0.0, 0.0)
|
||||
return 0.0, 0.0
|
||||
try:
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings
|
||||
verbose_proxy_logger.debug(
|
||||
"savings: no model info for provider=%s model=%s (%s)", custom_llm_provider, model, e
|
||||
)
|
||||
return _ModelRates(0.0, 0.0, 0.0, 0.0)
|
||||
return 0.0, 0.0
|
||||
input_cost = float(info.get("input_cost_per_token") or 0.0)
|
||||
output_cost = float(info.get("output_cost_per_token") or 0.0)
|
||||
cache_read = info.get("cache_read_input_token_cost")
|
||||
cache_write = info.get("cache_creation_input_token_cost")
|
||||
return _ModelRates(
|
||||
input=input_cost,
|
||||
output=output_cost,
|
||||
cache_read=input_cost if cache_read is None else float(cache_read),
|
||||
cache_write=input_cost if cache_write is None else float(cache_write),
|
||||
)
|
||||
cache_read_cost = info.get("cache_read_input_token_cost")
|
||||
if cache_read_cost is None:
|
||||
return input_cost, input_cost
|
||||
return input_cost, float(cache_read_cost)
|
||||
|
||||
|
||||
def _cost_of_usage(model: str, custom_llm_provider: str | None, usage: Usage) -> float | None:
|
||||
"""
|
||||
What ``usage`` costs on ``model``, or ``None`` when the model has no pricing.
|
||||
|
||||
Delegates to litellm's own cost engine rather than re-deriving per-token
|
||||
arithmetic, so cache-read and cache-creation tokens are split out of the
|
||||
inclusive ``prompt_tokens`` total exactly once, and tiered rates, ephemeral
|
||||
cache-write tiers and regional uplifts stay consistent with the spend the
|
||||
request was actually billed.
|
||||
"""
|
||||
try:
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model, usage=usage, custom_llm_provider=custom_llm_provider or ""
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings
|
||||
verbose_proxy_logger.debug(
|
||||
"savings: cannot price usage for provider=%s model=%s (%s)", custom_llm_provider, model, e
|
||||
)
|
||||
return None
|
||||
return prompt_cost + completion_cost
|
||||
|
||||
|
||||
def compute_autorouter_savings(
|
||||
|
|
@ -63,35 +74,45 @@ def compute_autorouter_savings(
|
|||
selected_model: str | None,
|
||||
baseline_provider: str | None,
|
||||
selected_provider: str | None,
|
||||
prompt_tokens: int,
|
||||
completion_tokens: int,
|
||||
cache_creation_input_tokens: int,
|
||||
usage: Usage,
|
||||
) -> float:
|
||||
"""
|
||||
Net dollars saved by routing this request to ``selected_model`` instead of the
|
||||
Net dollars saved by serving this request on ``selected_model`` instead of the
|
||||
counterfactual ``baseline_model``.
|
||||
|
||||
The model-switch delta prices prompt tokens at each model's input rate and
|
||||
completion tokens at each model's output rate (output is typically several
|
||||
times the input rate, so pricing completions at the input rate materially
|
||||
understates the gap). The cache-write penalty is the cost of switching: a
|
||||
cold cache on the selected deployment forces a cache-creation charge that the
|
||||
baseline would not have incurred, priced at the selected model's cache-write
|
||||
rate. Cache-read discounts are deliberately excluded here; they are attributed
|
||||
to the prompt-caching driver, so folding them in would double-count.
|
||||
Both arms price the same usage through litellm's cost engine, so the answer is
|
||||
the honest difference between what the request cost and what it would have cost
|
||||
on the baseline. Pricing the identical usage twice is what keeps the cache
|
||||
dimensions right: ``prompt_tokens`` already includes cache-read and
|
||||
cache-creation tokens, so charging them separately on top would count them
|
||||
twice, and the cost of a cold cache on the selected deployment is already
|
||||
inside its own arm at its own cache-creation rate.
|
||||
|
||||
Returns zero when routing did not change the model or when pricing is unknown.
|
||||
Floored at zero so an escalation to a pricier model never reads as negative
|
||||
savings on the dashboard.
|
||||
Returns zero when routing did not change the model or when either model has no
|
||||
pricing. Floored at zero so an escalation to a pricier model never reads as
|
||||
negative savings on the dashboard.
|
||||
"""
|
||||
if not baseline_model or not selected_model or baseline_model == selected_model:
|
||||
return 0.0
|
||||
baseline = _model_rates(baseline_model, baseline_provider)
|
||||
selected = _model_rates(selected_model, selected_provider)
|
||||
baseline_cost = (prompt_tokens * baseline.input) + (completion_tokens * baseline.output)
|
||||
selected_cost = (prompt_tokens * selected.input) + (completion_tokens * selected.output)
|
||||
cache_write_penalty = max(cache_creation_input_tokens, 0) * selected.cache_write
|
||||
return max((baseline_cost - selected_cost) - cache_write_penalty, 0.0)
|
||||
baseline_cost = _cost_of_usage(baseline_model, baseline_provider, usage)
|
||||
selected_cost = _cost_of_usage(selected_model, selected_provider, usage)
|
||||
if baseline_cost is None or selected_cost is None:
|
||||
return 0.0
|
||||
return max(baseline_cost - selected_cost, 0.0)
|
||||
|
||||
|
||||
def _usage_from_spend_log(usage_object: dict | None) -> Usage | None:
|
||||
"""
|
||||
Rebuild the request's ``Usage`` from the copy the spend log recorded, or
|
||||
``None`` when there is nothing priceable to rebuild it from.
|
||||
"""
|
||||
if not usage_object:
|
||||
return None
|
||||
try:
|
||||
return Usage(**usage_object)
|
||||
except Exception as e: # noqa: BLE001 # a malformed usage_object must not fail the daily spend write
|
||||
verbose_proxy_logger.debug("savings: unusable usage_object (%s)", e)
|
||||
return None
|
||||
|
||||
|
||||
def compute_savings_spend(
|
||||
|
|
@ -101,9 +122,7 @@ def compute_savings_spend(
|
|||
cache_read_input_tokens: int,
|
||||
baseline_model: str | None = None,
|
||||
baseline_provider: str | None = None,
|
||||
prompt_tokens: int = 0,
|
||||
completion_tokens: int = 0,
|
||||
cache_creation_input_tokens: int = 0,
|
||||
usage_object: dict | None = None,
|
||||
) -> SavingsSpend:
|
||||
"""
|
||||
Dollar savings for one request, split by optimization driver.
|
||||
|
|
@ -114,16 +133,19 @@ def compute_savings_spend(
|
|||
Auto-router savings compare the served ``model`` against the counterfactual
|
||||
``baseline_model`` and are zero unless the two differ.
|
||||
"""
|
||||
rates = _model_rates(model, custom_llm_provider)
|
||||
compression = max(compression_saved_tokens, 0) * rates.input
|
||||
prompt_caching = max(cache_read_input_tokens, 0) * max(rates.input - rates.cache_read, 0.0)
|
||||
input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider)
|
||||
compression = max(compression_saved_tokens, 0) * input_cost
|
||||
prompt_caching = max(cache_read_input_tokens, 0) * max(input_cost - cache_read_cost, 0.0)
|
||||
|
||||
usage = _usage_from_spend_log(usage_object)
|
||||
if usage is None or not model:
|
||||
return SavingsSpend(compression=compression, prompt_caching=prompt_caching)
|
||||
|
||||
autorouter = compute_autorouter_savings(
|
||||
baseline_model=baseline_model,
|
||||
selected_model=model,
|
||||
baseline_provider=baseline_provider,
|
||||
selected_provider=custom_llm_provider,
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
cache_creation_input_tokens=cache_creation_input_tokens,
|
||||
usage=usage,
|
||||
)
|
||||
return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter)
|
||||
|
|
|
|||
|
|
@ -11162,7 +11162,7 @@ class Router:
|
|||
|
||||
router_strategy = self._select_pre_routing_strategy(model=model, request_kwargs=request_kwargs)
|
||||
if router_strategy is None:
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
self._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None)
|
||||
return None
|
||||
|
||||
pre_routing_hook_response = await router_strategy.async_pre_routing_hook(
|
||||
|
|
@ -11174,11 +11174,7 @@ class Router:
|
|||
)
|
||||
self._record_routing_decision(
|
||||
request_kwargs=request_kwargs,
|
||||
routing_decision=(pre_routing_hook_response.routing_decision if pre_routing_hook_response else None),
|
||||
)
|
||||
self._record_savings_baseline_model(
|
||||
request_kwargs=request_kwargs,
|
||||
baseline_model=(pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None),
|
||||
pre_routing_hook_response=pre_routing_hook_response,
|
||||
)
|
||||
|
||||
# `model` (the alias, e.g. "smart-router") is never the deployment actually
|
||||
|
|
@ -11201,51 +11197,45 @@ class Router:
|
|||
@staticmethod
|
||||
def _record_routing_decision(
|
||||
request_kwargs: dict,
|
||||
routing_decision: StandardLoggingRoutingDecision | None,
|
||||
pre_routing_hook_response: Optional[PreRoutingHookResponse],
|
||||
) -> None:
|
||||
"""Make the request's metadata describe THIS routing attempt, and only this one.
|
||||
|
||||
Fallbacks re-enter the hook with the same `request_kwargs`, so an attempt that
|
||||
picks a plain model group after an auto-router group failed must clear the
|
||||
earlier decision; leaving it would attribute the first router's tier and cause
|
||||
to the deployment that actually served the request. Every attempt therefore
|
||||
writes or clears, never just writes.
|
||||
"""
|
||||
if routing_decision is None:
|
||||
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
|
||||
if isinstance(bucket, dict):
|
||||
bucket.pop("routing_decision", None)
|
||||
return
|
||||
to the deployment that actually served the request, and would price savings
|
||||
against a baseline this attempt never routed against. Every fact the hook
|
||||
records is therefore written or cleared here together, from one response, so
|
||||
no exit path can clear one and leave the other behind.
|
||||
|
||||
# `get_or_create_metadata_bucket` is the single owner of "which dict holds
|
||||
# proxy-internal metadata": it picks `litellm_metadata` when present (so the
|
||||
# decision never lands in the `metadata` dict that routes like /v1/messages
|
||||
# forward to the provider) and replaces a non-dict value rather than silently
|
||||
# skipping the write.
|
||||
_, metadata_bucket = get_or_create_metadata_bucket(request_kwargs)
|
||||
metadata_bucket["routing_decision"] = Router._redact_prompt_text_if_needed(
|
||||
request_kwargs=request_kwargs, routing_decision=routing_decision
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _record_savings_baseline_model(
|
||||
request_kwargs: dict,
|
||||
baseline_model: str | None,
|
||||
) -> None:
|
||||
"""Stash the auto-router's savings baseline for this attempt, same write-or-clear
|
||||
discipline as `_record_routing_decision`: a fallback that re-enters this hook
|
||||
without an auto-router strategy must clear a prior attempt's baseline, or the
|
||||
spend writer would price savings against a model this attempt never routed
|
||||
against.
|
||||
`get_or_create_metadata_bucket` is the single owner of "which dict holds
|
||||
proxy-internal metadata": it picks `litellm_metadata` when present (so nothing
|
||||
lands in the `metadata` dict that routes like /v1/messages forward to the
|
||||
provider) and replaces a non-dict value rather than silently skipping the write.
|
||||
"""
|
||||
if baseline_model is None:
|
||||
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
|
||||
if isinstance(bucket, dict):
|
||||
bucket.pop("auto_router_savings_baseline_model", None)
|
||||
routing_decision = pre_routing_hook_response.routing_decision if pre_routing_hook_response else None
|
||||
baseline_model = pre_routing_hook_response.savings_baseline_model if pre_routing_hook_response else None
|
||||
|
||||
recorded: dict[str, Any] = {}
|
||||
if routing_decision is not None:
|
||||
recorded["routing_decision"] = Router._redact_prompt_text_if_needed(
|
||||
request_kwargs=request_kwargs, routing_decision=routing_decision
|
||||
)
|
||||
if baseline_model is not None:
|
||||
recorded["auto_router_savings_baseline_model"] = baseline_model
|
||||
|
||||
cleared = {"routing_decision", "auto_router_savings_baseline_model"} - recorded.keys()
|
||||
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
|
||||
if isinstance(bucket, dict):
|
||||
for key in cleared:
|
||||
bucket.pop(key, None)
|
||||
|
||||
if not recorded:
|
||||
return
|
||||
|
||||
_, metadata_bucket = get_or_create_metadata_bucket(request_kwargs)
|
||||
metadata_bucket["auto_router_savings_baseline_model"] = baseline_model
|
||||
metadata_bucket.update(recorded)
|
||||
|
||||
@staticmethod
|
||||
def _redact_prompt_text_if_needed(
|
||||
|
|
|
|||
|
|
@ -20,9 +20,6 @@ else:
|
|||
|
||||
class AutoRouter(CustomLogger):
|
||||
DEFAULT_AUTO_SYNC_VALUE = "local"
|
||||
# Flagship the auto-router's savings are measured against when the deployment
|
||||
# does not configure `auto_router_savings_baseline_model`. Bare key (no
|
||||
# provider prefix) to match the pricing map's canonical entry.
|
||||
DEFAULT_SAVINGS_BASELINE_MODEL = "claude-opus-5"
|
||||
|
||||
def __init__(
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class SpendMetrics(BaseModel):
|
|||
compression_saved_tokens: int = Field(default=0)
|
||||
compression_savings_spend: float = Field(default=0.0)
|
||||
prompt_caching_savings_spend: float = Field(default=0.0)
|
||||
autorouter_savings_spend: float = Field(default=0.0)
|
||||
total_tokens: int = Field(default=0)
|
||||
successful_requests: int = Field(default=0)
|
||||
failed_requests: int = Field(default=0)
|
||||
|
|
@ -85,6 +86,7 @@ class DailySpendMetadata(BaseModel):
|
|||
total_compression_saved_tokens: int = Field(default=0)
|
||||
total_compression_savings_spend: float = Field(default=0.0)
|
||||
total_prompt_caching_savings_spend: float = Field(default=0.0)
|
||||
total_autorouter_savings_spend: float = Field(default=0.0)
|
||||
page: int = Field(default=1)
|
||||
total_pages: int = Field(default=1)
|
||||
has_more: bool = Field(default=False)
|
||||
|
|
@ -111,6 +113,7 @@ class LiteLLM_DailyUserSpend(BaseModel):
|
|||
compression_saved_tokens: int = 0
|
||||
compression_savings_spend: float = 0.0
|
||||
prompt_caching_savings_spend: float = 0.0
|
||||
autorouter_savings_spend: float = 0.0
|
||||
spend: float = 0.0
|
||||
api_requests: int = 0
|
||||
successful_requests: int = 0
|
||||
|
|
|
|||
|
|
@ -272,8 +272,6 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
auto_router_config: Optional[str] = None
|
||||
auto_router_default_model: Optional[str] = None
|
||||
auto_router_embedding_model: Optional[str] = None
|
||||
# counterfactual model the auto-router's savings are measured against on the
|
||||
# cost-optimization dashboard; unset falls back to AutoRouter.DEFAULT_SAVINGS_BASELINE_MODEL
|
||||
auto_router_savings_baseline_model: Optional[str] = None
|
||||
|
||||
# complexity-router params
|
||||
|
|
@ -843,9 +841,6 @@ class PreRoutingHookResponse(BaseModel):
|
|||
model: str
|
||||
messages: Optional[List[Dict[str, Any]]]
|
||||
routing_decision: StandardLoggingRoutingDecision | None = None
|
||||
# counterfactual model for the cost-optimization dashboard's auto-router savings
|
||||
# card; only auto-router populates this today, so it is None for every other
|
||||
# pre-routing strategy
|
||||
savings_baseline_model: Optional[str] = None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from litellm.proxy._types import (
|
|||
Litellm_EntityType,
|
||||
SpendUpdateQueueItem,
|
||||
)
|
||||
from litellm.proxy._types import BaseDailySpendTransaction
|
||||
from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import (
|
||||
DailySpendUpdateQueue,
|
||||
)
|
||||
|
|
@ -209,6 +210,7 @@ async def test_get_aggregated_daily_spend_update_transactions_same_key():
|
|||
"compression_saved_tokens": 0,
|
||||
"compression_savings_spend": 0,
|
||||
"prompt_caching_savings_spend": 0,
|
||||
"autorouter_savings_spend": 0,
|
||||
}
|
||||
|
||||
updates = [{test_key: test_transaction1}, {test_key: test_transaction2}]
|
||||
|
|
@ -259,6 +261,7 @@ async def test_flush_and_get_aggregated_daily_spend_update_transactions(
|
|||
"compression_saved_tokens": 0,
|
||||
"compression_savings_spend": 0,
|
||||
"prompt_caching_savings_spend": 0,
|
||||
"autorouter_savings_spend": 0,
|
||||
}
|
||||
|
||||
# Add updates to queue
|
||||
|
|
@ -527,3 +530,52 @@ async def test_compression_saved_tokens_aggregation(daily_spend_update_queue):
|
|||
assert agg["cache_creation_input_tokens"] == 7
|
||||
assert agg["compression_savings_spend"] == pytest.approx(0.0076)
|
||||
assert agg["prompt_caching_savings_spend"] == pytest.approx(0.0108)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_every_optional_daily_metric_aggregates(daily_spend_update_queue):
|
||||
"""Every additive metric must survive the merge, not just the ones wired by hand.
|
||||
|
||||
Two requests landing on one rollup key before a flush is the common case under
|
||||
load, and this same merge runs again on every cross-pod Redis drain. A metric
|
||||
persisted by the database write but skipped here is silently dropped on both
|
||||
paths, so the driver reads as zero on the dashboard however much it saved.
|
||||
"""
|
||||
test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic"
|
||||
numeric_fields = [
|
||||
name for name, annotation in BaseDailySpendTransaction.__annotations__.items() if annotation in (int, float)
|
||||
]
|
||||
assert "autorouter_savings_spend" in numeric_fields
|
||||
increments = {field: index + 1 for index, field in enumerate(numeric_fields)}
|
||||
|
||||
await daily_spend_update_queue.add_update({test_key: dict(increments)})
|
||||
await daily_spend_update_queue.add_update({test_key: dict(increments)})
|
||||
await daily_spend_update_queue.aggregate_queue_updates()
|
||||
updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue()
|
||||
|
||||
agg = updates[0][test_key]
|
||||
for field, value in increments.items():
|
||||
assert agg[field] == pytest.approx(value * 2), f"{field} did not accumulate"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_optional_metric_missing_from_an_older_payload_still_aggregates(
|
||||
daily_spend_update_queue,
|
||||
):
|
||||
"""A queued row written before a metric existed must not zero it out."""
|
||||
test_key = "user1_2023-01-01_key123_claude-haiku-4-5_anthropic"
|
||||
base = {
|
||||
"spend": 1.0,
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"api_requests": 1,
|
||||
"successful_requests": 1,
|
||||
"failed_requests": 0,
|
||||
}
|
||||
|
||||
await daily_spend_update_queue.add_update({test_key: dict(base)})
|
||||
await daily_spend_update_queue.add_update({test_key: {**base, "autorouter_savings_spend": 0.25}})
|
||||
await daily_spend_update_queue.aggregate_queue_updates()
|
||||
updates = await daily_spend_update_queue.flush_all_updates_from_in_memory_queue()
|
||||
|
||||
assert updates[0][test_key]["autorouter_savings_spend"] == pytest.approx(0.25)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ from litellm.proxy.management_endpoints.common_daily_activity import (
|
|||
get_daily_activity_aggregated,
|
||||
update_metrics,
|
||||
)
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import SpendMetrics
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
DailySpendMetadata,
|
||||
SpendMetrics,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -153,6 +156,7 @@ async def test_get_daily_activity_aggregated_with_endpoint_breakdown():
|
|||
"compression_saved_tokens": 0,
|
||||
"compression_savings_spend": 0.0,
|
||||
"prompt_caching_savings_spend": 0.0,
|
||||
"autorouter_savings_spend": 0.0,
|
||||
"failed_requests": 0,
|
||||
}
|
||||
mock_rows = [
|
||||
|
|
@ -498,6 +502,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
|
|||
mock_record_1.compression_saved_tokens = 0
|
||||
mock_record_1.compression_savings_spend = 0.0
|
||||
mock_record_1.prompt_caching_savings_spend = 0.0
|
||||
mock_record_1.autorouter_savings_spend = 0.0
|
||||
mock_record_1.api_requests = 10
|
||||
mock_record_1.successful_requests = 9
|
||||
mock_record_1.failed_requests = 1
|
||||
|
|
@ -520,6 +525,7 @@ async def test_tag_daily_activity_metadata_totals_not_zero():
|
|||
mock_record_2.compression_saved_tokens = 0
|
||||
mock_record_2.compression_savings_spend = 0.0
|
||||
mock_record_2.prompt_caching_savings_spend = 0.0
|
||||
mock_record_2.autorouter_savings_spend = 0.0
|
||||
mock_record_2.api_requests = 5
|
||||
mock_record_2.successful_requests = 5
|
||||
mock_record_2.failed_requests = 0
|
||||
|
|
@ -582,6 +588,7 @@ async def test_aggregated_activity_preserves_metadata_for_deleted_keys():
|
|||
"compression_saved_tokens": 0,
|
||||
"compression_savings_spend": 0.0,
|
||||
"prompt_caching_savings_spend": 0.0,
|
||||
"autorouter_savings_spend": 0.0,
|
||||
"failed_requests": 0,
|
||||
}
|
||||
mock_rows = [
|
||||
|
|
@ -669,6 +676,7 @@ def _daily_user_spend_record(*, user_id, api_key, spend, model="gpt-4", model_gr
|
|||
compression_saved_tokens=0,
|
||||
compression_savings_spend=0.0,
|
||||
prompt_caching_savings_spend=0.0,
|
||||
autorouter_savings_spend=0.0,
|
||||
api_requests=1,
|
||||
successful_requests=1,
|
||||
failed_requests=0,
|
||||
|
|
@ -973,6 +981,7 @@ async def test_get_daily_activity_aggregated_empty_result_set():
|
|||
"compression_saved_tokens": None,
|
||||
"compression_savings_spend": None,
|
||||
"prompt_caching_savings_spend": None,
|
||||
"autorouter_savings_spend": None,
|
||||
"api_requests": None,
|
||||
"successful_requests": None,
|
||||
"failed_requests": None,
|
||||
|
|
@ -1016,6 +1025,7 @@ def _no_spend_record():
|
|||
compression_saved_tokens=None,
|
||||
compression_savings_spend=None,
|
||||
prompt_caching_savings_spend=None,
|
||||
autorouter_savings_spend=None,
|
||||
api_requests=None,
|
||||
successful_requests=None,
|
||||
failed_requests=None,
|
||||
|
|
@ -1050,3 +1060,54 @@ def test_update_metrics_handles_none_values():
|
|||
assert metrics.cache_read_input_tokens == 0
|
||||
assert metrics.cache_creation_input_tokens == 0
|
||||
assert metrics.compression_saved_tokens == 0
|
||||
|
||||
|
||||
class TestEverySavingsDriverSurvivesTheReadPath:
|
||||
"""A savings driver is only real if it survives the whole read path.
|
||||
|
||||
The write path can price a driver correctly and persist it to all six rollup
|
||||
tables, and the dashboard can still render a permanent $0.00 because the
|
||||
aggregation query never summed the column or the response model never
|
||||
declared it. That failure is silent: the card renders, the number is just
|
||||
always zero, which is indistinguishable from having saved nothing. These
|
||||
tests enumerate the drivers from the response model itself, so a driver added
|
||||
later cannot be half-wired.
|
||||
"""
|
||||
|
||||
def _drivers(self) -> list[str]:
|
||||
drivers = [field for field in SpendMetrics.model_fields if field.endswith("_savings_spend")]
|
||||
assert drivers, "expected the dashboard response to expose at least one savings driver"
|
||||
return drivers
|
||||
|
||||
def test_every_driver_is_summed_by_the_rollup_query(self):
|
||||
sql, _ = _build_aggregated_sql_query(
|
||||
table_name="litellm_dailyuserspend",
|
||||
entity_id_field="user_id",
|
||||
entity_id="user-1",
|
||||
start_date="2026-07-01",
|
||||
end_date="2026-07-31",
|
||||
model=None,
|
||||
api_key=None,
|
||||
timezone_offset_minutes=None,
|
||||
)
|
||||
for driver in self._drivers():
|
||||
assert f"SUM({driver})" in sql, f"{driver} is never summed, so it reads as zero"
|
||||
|
||||
def test_every_driver_is_accumulated_across_rows(self):
|
||||
for driver in self._drivers():
|
||||
record = _no_spend_record()
|
||||
setattr(record, driver, 1.25)
|
||||
metrics = update_metrics(SpendMetrics(), record)
|
||||
assert getattr(metrics, driver) == pytest.approx(1.25), f"{driver} is dropped when accumulating rows"
|
||||
|
||||
def test_every_driver_is_carried_by_a_single_row_conversion(self):
|
||||
for driver in self._drivers():
|
||||
record = _no_spend_record()
|
||||
setattr(record, driver, 2.5)
|
||||
assert getattr(_record_to_spend_metrics(record), driver) == pytest.approx(2.5)
|
||||
|
||||
def test_every_driver_has_a_range_total(self):
|
||||
for driver in self._drivers():
|
||||
assert f"total_{driver}" in DailySpendMetadata.model_fields, (
|
||||
f"total_{driver} is missing, so the range summary omits the driver"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@ sys.path.insert(0, os.path.abspath("../../../.."))
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
|
||||
from litellm.proxy.spend_tracking.savings import (
|
||||
compute_autorouter_savings,
|
||||
compute_savings_spend,
|
||||
)
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
def _anthropic_costs(model: str) -> tuple[float, float]:
|
||||
|
|
@ -19,12 +21,37 @@ def _anthropic_costs(model: str) -> tuple[float, float]:
|
|||
return input_cost, cache_read_cost
|
||||
|
||||
|
||||
def _rates(model: str) -> tuple[float, float, float]:
|
||||
def _cached_usage_object() -> dict:
|
||||
"""A cache-heavy Anthropic request, shaped as the spend log records it.
|
||||
|
||||
`prompt_tokens` is the inclusive total: 3 uncached text tokens plus 500 read
|
||||
from cache plus 12304 written to cache.
|
||||
"""
|
||||
return {
|
||||
"prompt_tokens": 12807,
|
||||
"completion_tokens": 500,
|
||||
"total_tokens": 13307,
|
||||
"prompt_tokens_details": {"cached_tokens": 500, "cache_creation_tokens": 12304, "text_tokens": 3},
|
||||
"cache_creation_input_tokens": 12304,
|
||||
"cache_read_input_tokens": 500,
|
||||
}
|
||||
|
||||
|
||||
def _cost_on(model: str, usage_object: dict) -> float:
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model, usage=Usage(**usage_object), custom_llm_provider="anthropic"
|
||||
)
|
||||
return prompt_cost + completion_cost
|
||||
|
||||
|
||||
def _flat_rates(model: str) -> tuple[float, float, float]:
|
||||
info = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
|
||||
input_cost = info["input_cost_per_token"] or 0.0
|
||||
output_cost = info["output_cost_per_token"] or 0.0
|
||||
cache_write_cost = info.get("cache_creation_input_token_cost") or input_cost
|
||||
return input_cost, output_cost, cache_write_cost
|
||||
return (
|
||||
input_cost,
|
||||
info["output_cost_per_token"] or 0.0,
|
||||
info.get("cache_creation_input_token_cost") or input_cost,
|
||||
)
|
||||
|
||||
|
||||
def test_compression_savings_priced_at_input_rate():
|
||||
|
|
@ -89,70 +116,69 @@ def test_negative_token_counts_clamp_to_zero():
|
|||
assert result.prompt_caching == 0.0
|
||||
|
||||
|
||||
def test_autorouter_savings_prices_completion_at_output_rate():
|
||||
# Completion tokens must be priced at each model's OUTPUT rate, not its input
|
||||
# rate. Output is several times input on flagship models, so pricing
|
||||
# completions at the input rate materially understates the routed savings.
|
||||
base_in, base_out, _ = _rates("claude-opus-5")
|
||||
sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5")
|
||||
assert base_out > base_in # otherwise this test asserts nothing
|
||||
prompt_tokens, completion_tokens, cache_creation = 1000, 500, 200
|
||||
def test_autorouter_savings_does_not_double_charge_cache_tokens():
|
||||
"""`prompt_tokens` already includes cache-read and cache-creation tokens.
|
||||
|
||||
Charging those tokens again at the full input rate, or subtracting a separate
|
||||
cache-write penalty on top of them, prices the same tokens twice. Both arms go
|
||||
through litellm's cost engine on the identical usage, so each token is priced
|
||||
exactly once, in its own dimension.
|
||||
"""
|
||||
usage_object = _cached_usage_object()
|
||||
result = compute_autorouter_savings(
|
||||
baseline_model="claude-opus-5",
|
||||
selected_model="claude-haiku-4-5",
|
||||
baseline_provider="anthropic",
|
||||
selected_provider="anthropic",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
cache_creation_input_tokens=cache_creation,
|
||||
usage=Usage(**usage_object),
|
||||
)
|
||||
|
||||
baseline_cost = prompt_tokens * base_in + completion_tokens * base_out
|
||||
selected_cost = prompt_tokens * sel_in + completion_tokens * sel_out
|
||||
penalty = cache_creation * sel_cache_write
|
||||
assert result == pytest.approx(baseline_cost - selected_cost - penalty)
|
||||
expected = _cost_on("claude-opus-5", usage_object) - _cost_on("claude-haiku-4-5", usage_object)
|
||||
assert result == pytest.approx(expected)
|
||||
assert result > 0
|
||||
|
||||
# A mutant that priced completions at the input rate would land here instead.
|
||||
wrong = (prompt_tokens * base_in + completion_tokens * base_in) - (
|
||||
prompt_tokens * sel_in + completion_tokens * sel_in
|
||||
) - cache_creation * base_in
|
||||
assert result != pytest.approx(wrong)
|
||||
# The double-counting formula this replaced: every prompt token (cache reads
|
||||
# and cache writes included) charged at the flat input rate on both sides,
|
||||
# minus a cache-write penalty already accounted for inside the selected arm.
|
||||
base_in, base_out, base_write = _flat_rates("claude-opus-5")
|
||||
sel_in, sel_out, sel_write = _flat_rates("claude-haiku-4-5")
|
||||
prompt_tokens = usage_object["prompt_tokens"]
|
||||
completion_tokens = usage_object["completion_tokens"]
|
||||
double_counted = max(
|
||||
(prompt_tokens * base_in + completion_tokens * base_out)
|
||||
- (prompt_tokens * sel_in + completion_tokens * sel_out)
|
||||
- usage_object["cache_creation_input_tokens"] * sel_write,
|
||||
0.0,
|
||||
)
|
||||
assert result != pytest.approx(double_counted)
|
||||
|
||||
|
||||
def test_autorouter_cache_write_penalty_uses_selected_model_rate():
|
||||
# The switch penalty is a cache-creation charge on the deployment actually
|
||||
# written to (the selected model), priced at its cache-creation rate, not the
|
||||
# baseline's input rate.
|
||||
base_in, base_out, _ = _rates("claude-opus-5")
|
||||
sel_in, sel_out, sel_cache_write = _rates("claude-haiku-4-5")
|
||||
prompt_tokens, completion_tokens, cache_creation = 0, 0, 1000
|
||||
|
||||
def test_autorouter_savings_charges_cache_reads_at_the_cache_read_rate():
|
||||
"""A request served almost entirely from cache is cheap on both models, so the
|
||||
routed saving must be far smaller than the same token count would suggest at
|
||||
full input price."""
|
||||
usage_object = {
|
||||
"prompt_tokens": 10_000,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 10_000,
|
||||
"prompt_tokens_details": {"cached_tokens": 10_000, "text_tokens": 0},
|
||||
"cache_read_input_tokens": 10_000,
|
||||
}
|
||||
result = compute_autorouter_savings(
|
||||
baseline_model="claude-opus-5",
|
||||
selected_model="claude-haiku-4-5",
|
||||
baseline_provider="anthropic",
|
||||
selected_provider="anthropic",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
cache_creation_input_tokens=cache_creation,
|
||||
usage=Usage(**usage_object),
|
||||
)
|
||||
# With no prompt/completion tokens, savings is purely the negative penalty,
|
||||
# floored at zero, so a pure-penalty request never reads as savings.
|
||||
assert result == 0.0
|
||||
|
||||
# Confirm the penalty magnitude uses the SELECTED model's cache-write rate by
|
||||
# giving enough token delta to stay positive, then isolating the penalty.
|
||||
prompt_tokens = 100_000
|
||||
with_penalty = compute_autorouter_savings(
|
||||
"claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic",
|
||||
prompt_tokens, 0, cache_creation,
|
||||
)
|
||||
without_penalty = compute_autorouter_savings(
|
||||
"claude-opus-5", "claude-haiku-4-5", "anthropic", "anthropic",
|
||||
prompt_tokens, 0, 0,
|
||||
)
|
||||
assert without_penalty - with_penalty == pytest.approx(cache_creation * sel_cache_write)
|
||||
base_read = litellm.get_model_info("claude-opus-5", "anthropic")["cache_read_input_token_cost"]
|
||||
sel_read = litellm.get_model_info("claude-haiku-4-5", "anthropic")["cache_read_input_token_cost"]
|
||||
assert result == pytest.approx(10_000 * (base_read - sel_read))
|
||||
|
||||
base_in = litellm.get_model_info("claude-opus-5", "anthropic")["input_cost_per_token"]
|
||||
sel_in = litellm.get_model_info("claude-haiku-4-5", "anthropic")["input_cost_per_token"]
|
||||
assert result < 10_000 * (base_in - sel_in)
|
||||
|
||||
|
||||
def test_autorouter_savings_zero_when_model_unchanged():
|
||||
|
|
@ -161,9 +187,7 @@ def test_autorouter_savings_zero_when_model_unchanged():
|
|||
selected_model="claude-opus-5",
|
||||
baseline_provider="anthropic",
|
||||
selected_provider="anthropic",
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=500,
|
||||
cache_creation_input_tokens=0,
|
||||
usage=Usage(**_cached_usage_object()),
|
||||
)
|
||||
assert result == 0.0
|
||||
|
||||
|
|
@ -175,9 +199,18 @@ def test_autorouter_savings_floored_at_zero_on_escalation():
|
|||
selected_model="claude-opus-5",
|
||||
baseline_provider="anthropic",
|
||||
selected_provider="anthropic",
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=500,
|
||||
cache_creation_input_tokens=0,
|
||||
usage=Usage(**_cached_usage_object()),
|
||||
)
|
||||
assert result == 0.0
|
||||
|
||||
|
||||
def test_autorouter_savings_unknown_baseline_fails_open_to_zero():
|
||||
result = compute_autorouter_savings(
|
||||
baseline_model="totally-made-up-model-xyz",
|
||||
selected_model="claude-haiku-4-5",
|
||||
baseline_provider="anthropic",
|
||||
selected_provider="anthropic",
|
||||
usage=Usage(**_cached_usage_object()),
|
||||
)
|
||||
assert result == 0.0
|
||||
|
||||
|
|
@ -190,17 +223,13 @@ def test_autorouter_savings_zero_without_baseline():
|
|||
compression_saved_tokens=0,
|
||||
cache_read_input_tokens=0,
|
||||
baseline_model=None,
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=500,
|
||||
usage_object=_cached_usage_object(),
|
||||
)
|
||||
assert result.autorouter == 0.0
|
||||
|
||||
|
||||
def test_compute_savings_spend_includes_autorouter_driver():
|
||||
base_in, base_out, _ = _rates("claude-opus-5")
|
||||
sel_in, sel_out, _ = _rates("claude-haiku-4-5")
|
||||
prompt_tokens, completion_tokens = 2000, 800
|
||||
|
||||
usage_object = _cached_usage_object()
|
||||
result = compute_savings_spend(
|
||||
model="claude-haiku-4-5",
|
||||
custom_llm_provider="anthropic",
|
||||
|
|
@ -208,12 +237,24 @@ def test_compute_savings_spend_includes_autorouter_driver():
|
|||
cache_read_input_tokens=0,
|
||||
baseline_model="claude-opus-5",
|
||||
baseline_provider="anthropic",
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
cache_creation_input_tokens=0,
|
||||
)
|
||||
expected = (prompt_tokens * base_in + completion_tokens * base_out) - (
|
||||
prompt_tokens * sel_in + completion_tokens * sel_out
|
||||
usage_object=usage_object,
|
||||
)
|
||||
expected = _cost_on("claude-opus-5", usage_object) - _cost_on("claude-haiku-4-5", usage_object)
|
||||
assert result.autorouter == pytest.approx(expected)
|
||||
assert result.autorouter > 0
|
||||
|
||||
|
||||
def test_compute_savings_spend_without_usage_object_keeps_other_drivers():
|
||||
"""A row with no recorded usage still prices compression and caching; only the
|
||||
counterfactual driver needs the usage breakdown."""
|
||||
input_cost, _ = _anthropic_costs("claude-sonnet-5")
|
||||
result = compute_savings_spend(
|
||||
model="claude-sonnet-5",
|
||||
custom_llm_provider="anthropic",
|
||||
compression_saved_tokens=1000,
|
||||
cache_read_input_tokens=0,
|
||||
baseline_model="claude-opus-5",
|
||||
usage_object=None,
|
||||
)
|
||||
assert result.compression == pytest.approx(1000 * input_cost)
|
||||
assert result.autorouter == 0.0
|
||||
|
|
|
|||
|
|
@ -2437,7 +2437,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
@ -2533,7 +2533,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
@ -2627,7 +2627,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ from litellm.router_strategy.complexity_router.config import (
|
|||
from litellm.types.router import (
|
||||
Deployment,
|
||||
LiteLLM_Params,
|
||||
PreRoutingHookResponse,
|
||||
TaggedPreRoutingStrategy,
|
||||
)
|
||||
|
||||
|
|
@ -4081,22 +4082,61 @@ class TestRecordRoutingDecision:
|
|||
the request's metadata must describe the current attempt and nothing else."""
|
||||
|
||||
DECISION = {"router_model_name": "smart-router", "router_type": "complexity", "routed_model": "gpt-4o-mini"}
|
||||
STALE_METADATA_KEYS = ("routing_decision", "auto_router_savings_baseline_model")
|
||||
|
||||
def test_none_clears_a_previous_decision_from_both_buckets(self):
|
||||
request_kwargs: Dict = {
|
||||
"metadata": {"routing_decision": self.DECISION, "keep": 1},
|
||||
"litellm_metadata": {"routing_decision": self.DECISION},
|
||||
}
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None)
|
||||
assert "routing_decision" not in request_kwargs["metadata"]
|
||||
assert "routing_decision" not in request_kwargs["litellm_metadata"]
|
||||
assert request_kwargs["metadata"]["keep"] == 1
|
||||
|
||||
def test_none_creates_no_bucket_on_a_request_that_had_none(self):
|
||||
request_kwargs: Dict = {}
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None)
|
||||
assert request_kwargs == {}
|
||||
|
||||
@pytest.mark.parametrize("stale_key", STALE_METADATA_KEYS)
|
||||
def test_an_attempt_without_a_strategy_clears_every_stale_fact(self, stale_key):
|
||||
"""A fallback to a plain model group re-enters the hook with the same
|
||||
`request_kwargs`. Anything the auto-router attempt left behind would be
|
||||
attributed to the deployment that actually served the request, letting a
|
||||
caller who forces a router failure inflate the recorded savings."""
|
||||
request_kwargs: Dict = {
|
||||
"metadata": {stale_key: "stale", "keep": 1},
|
||||
"litellm_metadata": {stale_key: "stale"},
|
||||
}
|
||||
Router._record_routing_decision(request_kwargs=request_kwargs, pre_routing_hook_response=None)
|
||||
assert stale_key not in request_kwargs["metadata"]
|
||||
assert stale_key not in request_kwargs["litellm_metadata"]
|
||||
assert request_kwargs["metadata"]["keep"] == 1
|
||||
|
||||
def test_a_response_without_a_baseline_clears_a_previous_one(self):
|
||||
"""Not every pre-routing strategy sets a savings baseline; one that does not
|
||||
must not inherit the previous attempt's."""
|
||||
request_kwargs: Dict = {"litellm_metadata": {"auto_router_savings_baseline_model": "claude-opus-5"}}
|
||||
Router._record_routing_decision(
|
||||
request_kwargs=request_kwargs,
|
||||
pre_routing_hook_response=PreRoutingHookResponse(model="gpt-4o-mini", messages=[]),
|
||||
)
|
||||
assert "auto_router_savings_baseline_model" not in request_kwargs["litellm_metadata"]
|
||||
|
||||
def test_baseline_is_recorded_on_the_internal_bucket(self):
|
||||
"""The baseline must land in `litellm_metadata`, never the `metadata` dict that
|
||||
surfaces like /v1/messages forward verbatim to the provider."""
|
||||
request_kwargs: Dict = {"litellm_metadata": {}, "metadata": {}}
|
||||
Router._record_routing_decision(
|
||||
request_kwargs=request_kwargs,
|
||||
pre_routing_hook_response=PreRoutingHookResponse(
|
||||
model="claude-haiku-4-5", messages=[], savings_baseline_model="claude-opus-5"
|
||||
),
|
||||
)
|
||||
assert request_kwargs["litellm_metadata"]["auto_router_savings_baseline_model"] == "claude-opus-5"
|
||||
assert "auto_router_savings_baseline_model" not in request_kwargs["metadata"]
|
||||
|
||||
|
||||
class TestEscalationIsRecordedConsistently:
|
||||
"""An escalation keyword records two separate facts on every path: that the caller
|
||||
|
|
|
|||
10
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
10
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -23487,6 +23487,11 @@ export interface components {
|
|||
* @default 0
|
||||
*/
|
||||
total_api_requests: number;
|
||||
/**
|
||||
* Total Autorouter Savings Spend
|
||||
* @default 0
|
||||
*/
|
||||
total_autorouter_savings_spend: number;
|
||||
/**
|
||||
* Total Cache Creation Input Tokens
|
||||
* @default 0
|
||||
|
|
@ -31390,6 +31395,11 @@ export interface components {
|
|||
* @default 0
|
||||
*/
|
||||
api_requests: number;
|
||||
/**
|
||||
* Autorouter Savings Spend
|
||||
* @default 0
|
||||
*/
|
||||
autorouter_savings_spend: number;
|
||||
/**
|
||||
* Cache Creation Input Tokens
|
||||
* @default 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue