mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
feat(spend): tell a mid-conversation switch from a conversation's first turn
The auto-router savings number reads a cold cache the same way whichever reason it is cold for, and the two want opposite arithmetic. A switch leaves the new model cold, so the write is a cost the switch caused and the baseline, already warm on the model it never left, would have paid only a read. A first turn has nothing cached anywhere, so the baseline would have written the same prompt and the write belongs on both arms. Charging every cold cache as a switch, which is all the rollup row could support, understated a genuine first turn badly: on a 20k prompt with a 1k completion it reported +$0.005 against a true +$0.12, and a whole conversation only converged as it lengthened, reaching half its real value at five turns. Worse, the write premium is fixed by prompt size while the saving grows with completion length, so under roughly 750 completion tokens on a 20k prompt the understated number crossed zero and a profitable route rendered as a loss. The discriminator is what the session was served last. The complexity router records the model it picked against the proxy-derived session id, reads it back on the next turn and carries it on the routing decision, where the existing per-attempt writer records it with the baseline so a fallback clears both together. Three states, not two: no session at all keeps the conservative rule and under-claims, a tracked session with nothing recorded is a real first turn, and a tracked session served something else before is the switch. Session identity moves to router_utils/session_identity.py, since the router now needs the same reader the complexity router already had. That reader is wider than its docstring claimed: the proxy writes the id it derives from an `x-*-session-id` header or from Anthropic's `metadata.user_id` into the same `metadata.session_id` the complexity router was reading, so header-carrying clients were always covered. Observation only. Nothing is pinned, no candidate pool narrows, and `session_affinity` is untouched; a request naming no session never touches the cache at all.
This commit is contained in:
Tin Chi Lo
parent
bfe34130ed
commit
db33f45641
13 changed files with 473 additions and 44 deletions
|
|
@ -1655,3 +1655,10 @@ ADVISOR_TOOL_DESCRIPTION: str = (
|
|||
"want to verify your reasoning, or face a complex decision. "
|
||||
"Describe your question or challenge clearly in the 'question' field."
|
||||
)
|
||||
|
||||
########################### AUTO-ROUTER SAVINGS CONSTANTS ###########################
|
||||
# How long the model served to a session is remembered, so a later turn can tell a
|
||||
# model switch from a conversation that just started. Matches the session-affinity
|
||||
# pin's default lifetime; a conversation quieter than this is treated as new, which
|
||||
# falls back to the conservative savings rule rather than misattributing a switch.
|
||||
AUTOROUTER_PREVIOUS_MODEL_TTL_SECONDS: int = 3600
|
||||
|
|
|
|||
|
|
@ -3322,6 +3322,8 @@ class SpendLogsMetadata(TypedDict):
|
|||
cost_breakdown: Optional[CostBreakdown] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
|
||||
compression_savings: CompressionSavingsMetadata | None
|
||||
auto_router_savings_baseline_model: str | None # counterfactual model for the auto-router savings driver
|
||||
auto_router_previous_model: str | None # model this session was served last; tells a switch from a first turn
|
||||
auto_router_session_tracked: bool | None # whether the request named a session at all
|
||||
|
||||
|
||||
class SpendLogsPayload(TypedDict):
|
||||
|
|
|
|||
|
|
@ -1869,6 +1869,8 @@ class DBSpendUpdateWriter:
|
|||
compression_saved_tokens=compression_saved_tokens,
|
||||
cache_read_input_tokens=cache_read_input_tokens,
|
||||
baseline_model=_metadata.get("auto_router_savings_baseline_model"),
|
||||
previous_model=_metadata.get("auto_router_previous_model"),
|
||||
session_tracked=bool(_metadata.get("auto_router_session_tracked")),
|
||||
usage_object=usage_obj,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -204,6 +204,8 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS = (
|
|||
"policy_sources",
|
||||
"routing_decision",
|
||||
"auto_router_savings_baseline_model",
|
||||
"auto_router_previous_model",
|
||||
"auto_router_session_tracked",
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
"standard_logging_object",
|
||||
"proxy_server_request",
|
||||
|
|
|
|||
|
|
@ -103,16 +103,36 @@ _CACHE_SPLIT_FIELDS = frozenset(
|
|||
)
|
||||
|
||||
|
||||
def _baseline_usage(usage: Usage) -> Usage:
|
||||
def _is_mid_conversation_switch(
|
||||
previous_model: _ModelIdentity | None, selected: _ModelIdentity, session_tracked: bool
|
||||
) -> bool:
|
||||
"""Whether some other model was already warm for this conversation.
|
||||
|
||||
Three states, not two. An untracked request named no session at all, so nothing
|
||||
can say why its cache is cold and it stays on the conservative rule, which
|
||||
under-claims rather than inflates. A tracked request with no previous model is a
|
||||
genuine first turn, and one whose previous model is the selected one stayed put;
|
||||
neither is a switch, and both would have paid the same write on a single-model
|
||||
deployment. Only a tracked request served something else before is.
|
||||
"""
|
||||
if not session_tracked:
|
||||
return True
|
||||
return previous_model is not None and previous_model != selected
|
||||
|
||||
|
||||
def _baseline_usage(usage: Usage, is_switch: bool) -> Usage:
|
||||
"""The same request as a single-model baseline would have met it.
|
||||
|
||||
Staying on one model, the prompt is written to cache once and read from thereafter,
|
||||
so whatever this request paid to write would already have been cached on the
|
||||
baseline. That holds whether or not this request also read anything: a switch to a
|
||||
cold model reads nothing precisely because its cache is empty, which is the case the
|
||||
penalty exists for. Gating on a read instead would charge the baseline a write it
|
||||
would never repeat, and a cold switch would then report a larger saving than the
|
||||
same traffic with caching turned off.
|
||||
On a switch, the model that was already serving this conversation had the prompt
|
||||
cached, so whatever this request paid to write would have been a read on the
|
||||
baseline. The write is the switch's own cost and has to count against the saving,
|
||||
which is why the cache tokens move into the read bucket here.
|
||||
|
||||
On a first turn nothing was cached anywhere, so the baseline would have paid the
|
||||
same write; leaving the usage alone lets both arms carry it and the saving comes
|
||||
out as the rate difference it really is. Charging the write to both cases, which
|
||||
is what having no discriminator forces, understates a genuine first turn to a
|
||||
few percent of its value and can render it as a loss.
|
||||
|
||||
Only the cache buckets move. Every other field the request was priced on travels
|
||||
through untouched, audio and image and video counts among them, because the baseline
|
||||
|
|
@ -122,7 +142,7 @@ def _baseline_usage(usage: Usage) -> Usage:
|
|||
"""
|
||||
cache_read, cache_creation = _cache_token_split(usage)
|
||||
details = usage.prompt_tokens_details
|
||||
if details is None or cache_creation <= 0:
|
||||
if details is None or cache_creation <= 0 or not is_switch:
|
||||
return usage
|
||||
return Usage(
|
||||
prompt_tokens=usage.prompt_tokens,
|
||||
|
|
@ -149,6 +169,8 @@ def compute_autorouter_savings(
|
|||
selected_model: str | None,
|
||||
selected_provider: str | None,
|
||||
usage: Usage,
|
||||
previous_model: str | None = None,
|
||||
session_tracked: bool = False,
|
||||
) -> float:
|
||||
"""Net dollars the router saved, or cost, by serving this request on ``selected_model``.
|
||||
|
||||
|
|
@ -157,6 +179,11 @@ def compute_autorouter_savings(
|
|||
incurred; when that charge outweighs the cheaper rates, routing lost money and the
|
||||
dashboard has to be able to say so. Zero when both sides resolve to the same
|
||||
deployment, or when either cannot be resolved or priced.
|
||||
|
||||
``previous_model`` is what this conversation was served last and ``session_tracked``
|
||||
whether the request named a session at all. Together they separate a switch from a
|
||||
first turn; without a session there is no discriminator, so every cold cache is
|
||||
charged as a switch, which understates rather than inflates.
|
||||
"""
|
||||
# No provider argument for the baseline on purpose: it arrives from the routing
|
||||
# metadata as a single self-describing string, already qualified by the auto-router,
|
||||
|
|
@ -165,7 +192,11 @@ def compute_autorouter_savings(
|
|||
selected = _resolve_model(selected_model, selected_provider)
|
||||
if baseline is None or selected is None or baseline == selected:
|
||||
return 0.0
|
||||
baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage))
|
||||
# Resolved, not string-compared: the previous model is recorded as the router's own
|
||||
# model-group name while the selected one arrives normalized from the spend log, so
|
||||
# raw equality reads `anthropic/claude-opus-5` as a switch away from `claude-opus-5`.
|
||||
is_switch = _is_mid_conversation_switch(_resolve_model(previous_model, None), selected, session_tracked)
|
||||
baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, is_switch=is_switch))
|
||||
selected_cost = _cost_of_usage(selected, usage)
|
||||
if baseline_cost is None or selected_cost is None:
|
||||
return 0.0
|
||||
|
|
@ -193,6 +224,8 @@ def compute_savings_spend(
|
|||
cache_read_input_tokens: int,
|
||||
baseline_model: str | None = None,
|
||||
usage_object: dict | None = None,
|
||||
previous_model: str | None = None,
|
||||
session_tracked: bool = False,
|
||||
) -> SavingsSpend:
|
||||
"""
|
||||
Dollar savings for one request, split by optimization driver.
|
||||
|
|
@ -201,7 +234,8 @@ def compute_savings_spend(
|
|||
input rate. Prompt-caching savings price the cache-read tokens at the
|
||||
difference between the input rate and the discounted cache-read rate.
|
||||
Auto-router savings compare the served ``model`` against the counterfactual
|
||||
``baseline_model`` and are zero unless the two differ.
|
||||
``baseline_model`` and are zero unless the two differ; ``previous_model``
|
||||
tells a mid-conversation switch from a conversation's first turn.
|
||||
"""
|
||||
input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider)
|
||||
compression = max(compression_saved_tokens, 0) * input_cost
|
||||
|
|
@ -215,6 +249,8 @@ def compute_savings_spend(
|
|||
baseline_model=baseline_model,
|
||||
selected_model=model,
|
||||
selected_provider=custom_llm_provider,
|
||||
previous_model=previous_model,
|
||||
session_tracked=session_tracked,
|
||||
usage=usage,
|
||||
)
|
||||
return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter)
|
||||
|
|
|
|||
|
|
@ -118,6 +118,8 @@ def _get_spend_logs_metadata(
|
|||
cost_breakdown=None,
|
||||
compression_savings=None,
|
||||
auto_router_savings_baseline_model=None,
|
||||
auto_router_previous_model=None,
|
||||
auto_router_session_tracked=None,
|
||||
litellm_call_id=litellm_call_id,
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
|
|||
|
|
@ -11224,8 +11224,21 @@ class Router:
|
|||
)
|
||||
if baseline_model is not None:
|
||||
recorded["auto_router_savings_baseline_model"] = baseline_model
|
||||
previous_model = pre_routing_hook_response.previous_model if pre_routing_hook_response else None
|
||||
if previous_model is not None:
|
||||
recorded["auto_router_previous_model"] = previous_model
|
||||
# Recorded even when there is no previous model: a tracked session with none is a
|
||||
# genuine first turn, which is priced the opposite way to a request that named no
|
||||
# session at all. Collapsing the two into a missing key loses that distinction.
|
||||
if pre_routing_hook_response is not None and pre_routing_hook_response.session_tracked:
|
||||
recorded["auto_router_session_tracked"] = True
|
||||
|
||||
cleared = {"routing_decision", "auto_router_savings_baseline_model"} - recorded.keys()
|
||||
cleared = {
|
||||
"routing_decision",
|
||||
"auto_router_savings_baseline_model",
|
||||
"auto_router_previous_model",
|
||||
"auto_router_session_tracked",
|
||||
} - recorded.keys()
|
||||
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
|
||||
if isinstance(bucket, dict):
|
||||
for key in cleared:
|
||||
|
|
|
|||
|
|
@ -25,7 +25,11 @@ from typing import TYPE_CHECKING, Any, Literal, NamedTuple, Union, cast
|
|||
from pydantic import BaseModel
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY
|
||||
from litellm.constants import (
|
||||
AUTOROUTER_PREVIOUS_MODEL_TTL_SECONDS,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
RETURN_RAW_MODEL_NAME_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.base_llm.base_utils import type_to_response_format_param
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -1293,35 +1297,26 @@ class ComplexityRouter(CustomLogger):
|
|||
"""
|
||||
return _extract_current_ask_and_system_prompt(messages)
|
||||
|
||||
@staticmethod
|
||||
def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]:
|
||||
"""Metadata may land on `metadata` or `litellm_metadata` depending on the
|
||||
endpoint, mirroring DeploymentAffinityCheck's precedence."""
|
||||
return [
|
||||
metadata
|
||||
for metadata_key in ("litellm_metadata", "metadata")
|
||||
if isinstance(metadata := request_kwargs.get(metadata_key), dict)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
|
||||
"""Resolve a client-supplied session_id."""
|
||||
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
|
||||
session_id = metadata.get("session_id")
|
||||
if session_id is not None:
|
||||
return str(session_id)
|
||||
return None
|
||||
"""Resolve the session this request belongs to.
|
||||
|
||||
Wider than a client-supplied `metadata.session_id`: the proxy also writes the
|
||||
session id it derives from an `x-*-session-id` header or from Anthropic's
|
||||
`metadata.user_id` into the same field before routing.
|
||||
"""
|
||||
from litellm.router_utils.session_identity import session_id_from_request
|
||||
|
||||
return session_id_from_request(request_kwargs)
|
||||
|
||||
@staticmethod
|
||||
def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
|
||||
"""Resolve the proxy-derived API key hash, the same trust boundary
|
||||
DeploymentAffinityCheck uses for its own key-based affinity (not the
|
||||
client-supplied OpenAI `user` param, which isn't authenticated)."""
|
||||
for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
|
||||
user_key = metadata.get("user_api_key_hash")
|
||||
if user_key is not None:
|
||||
return str(user_key)
|
||||
return None
|
||||
from litellm.router_utils.session_identity import user_api_key_hash_from_request
|
||||
|
||||
return user_api_key_hash_from_request(request_kwargs)
|
||||
|
||||
def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str:
|
||||
# Namespace by the caller's API key hash so two different callers reusing the
|
||||
|
|
@ -1331,6 +1326,54 @@ class ComplexityRouter(CustomLogger):
|
|||
caller_scope = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped"
|
||||
return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}"
|
||||
|
||||
def _previous_model_cache_key(self, request_kwargs: dict) -> str | None:
|
||||
"""Where this session's last-served model is remembered, or ``None`` if it names no session.
|
||||
|
||||
Separate from the session-affinity key on purpose: this is recorded on every
|
||||
request whether or not affinity is enabled, and the two must not read each
|
||||
other's values.
|
||||
"""
|
||||
from litellm.router_utils.session_identity import session_scope
|
||||
|
||||
return session_scope(
|
||||
request_kwargs, namespace=self.model_name, discriminator="complexity_router_previous_model"
|
||||
)
|
||||
|
||||
async def _previous_model_for_session(self, cache_key: str | None) -> str | None:
|
||||
"""The model this conversation was served last, or ``None`` if it is new.
|
||||
|
||||
Savings are priced against a counterfactual single-model deployment, and a cold
|
||||
cache alone cannot say whether this request is cold because the router switched
|
||||
models or because the conversation just started. Those two want opposite
|
||||
arithmetic, so what the session was served last is the discriminator: different
|
||||
from the model picked now means the cache write is the switch's own cost, absent
|
||||
or equal means a single-model deployment would have paid it too.
|
||||
|
||||
Observation only. Nothing read here pins a deployment or narrows the candidate
|
||||
pool; `session_affinity` is a separate feature and is unaffected.
|
||||
"""
|
||||
if cache_key is None:
|
||||
return None
|
||||
try:
|
||||
previous_model = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
|
||||
except Exception as e: # noqa: BLE001 # a dashboard metric must not fail a live request
|
||||
verbose_router_logger.debug("complexity router: could not read the session's previous model (%s)", e)
|
||||
return None
|
||||
return previous_model if isinstance(previous_model, str) else None
|
||||
|
||||
async def _remember_model_for_session(self, cache_key: str | None, routed_model: str) -> None:
|
||||
"""Record what this session was served, for the next turn to compare against."""
|
||||
if cache_key is None:
|
||||
return
|
||||
try:
|
||||
await self.litellm_router_instance.cache.async_set_cache(
|
||||
key=cache_key,
|
||||
value=routed_model,
|
||||
ttl=AUTOROUTER_PREVIOUS_MODEL_TTL_SECONDS,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # a dashboard metric must not fail a live request
|
||||
verbose_router_logger.debug("complexity router: could not record the session's model (%s)", e)
|
||||
|
||||
async def async_pre_routing_hook(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -1359,6 +1402,9 @@ class ComplexityRouter(CustomLogger):
|
|||
if isinstance(metadata, dict):
|
||||
metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True
|
||||
|
||||
previous_model_key = self._previous_model_cache_key(request_kwargs)
|
||||
previous_model = await self._previous_model_for_session(previous_model_key)
|
||||
|
||||
use_session_affinity = self.config.session_affinity and not self.config.plugins
|
||||
session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
|
||||
cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
|
||||
|
|
@ -1397,10 +1443,13 @@ class ComplexityRouter(CustomLogger):
|
|||
f"ComplexityRouter: routing decision cause={cause}, routed_model={routed_model}"
|
||||
)
|
||||
has_original_messages = messages is not None and len(messages) > 0
|
||||
await self._remember_model_for_session(previous_model_key, routed_model)
|
||||
return PreRoutingHookResponse(
|
||||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
previous_model=previous_model,
|
||||
session_tracked=previous_model_key is not None,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=cause,
|
||||
|
|
@ -1415,7 +1464,11 @@ class ComplexityRouter(CustomLogger):
|
|||
messages=messages,
|
||||
input=input,
|
||||
specific_deployment=specific_deployment,
|
||||
previous_model=previous_model,
|
||||
session_tracked=previous_model_key is not None,
|
||||
)
|
||||
if response is not None:
|
||||
await self._remember_model_for_session(previous_model_key, response.model)
|
||||
if cache_key is not None and response is not None:
|
||||
await self.litellm_router_instance.cache.async_set_cache(
|
||||
key=cache_key,
|
||||
|
|
@ -1431,6 +1484,8 @@ class ComplexityRouter(CustomLogger):
|
|||
messages: list[dict[str, Any]] | None = None,
|
||||
input: Union[str, list] | None = None,
|
||||
specific_deployment: bool | None = False,
|
||||
previous_model: str | None = None,
|
||||
session_tracked: bool = False,
|
||||
) -> PreRoutingHookResponse | None:
|
||||
"""
|
||||
Classifies the request by complexity and returns the appropriate model.
|
||||
|
|
@ -1478,6 +1533,8 @@ class ComplexityRouter(CustomLogger):
|
|||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
previous_model=previous_model,
|
||||
session_tracked=session_tracked,
|
||||
routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"),
|
||||
)
|
||||
|
||||
|
|
@ -1500,6 +1557,8 @@ class ComplexityRouter(CustomLogger):
|
|||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
previous_model=previous_model,
|
||||
session_tracked=session_tracked,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=keyword_cause,
|
||||
|
|
@ -1548,6 +1607,8 @@ class ComplexityRouter(CustomLogger):
|
|||
model=routed_model,
|
||||
messages=messages if has_original_messages else None,
|
||||
savings_baseline_model=self.savings_baseline_model,
|
||||
previous_model=previous_model,
|
||||
session_tracked=session_tracked,
|
||||
routing_decision=self._build_routing_decision(
|
||||
routed_model=routed_model,
|
||||
cause=outcome.cause,
|
||||
|
|
|
|||
54
litellm/router_utils/session_identity.py
Normal file
54
litellm/router_utils/session_identity.py
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
"""Reading the caller and conversation a request belongs to, off its metadata.
|
||||
|
||||
The proxy derives both before routing and drops them on the request: any
|
||||
``x-*-session-id`` or trace-id header, and Anthropic's ``metadata.user_id``,
|
||||
both land in ``metadata["session_id"]`` (``litellm_pre_call_utils``), while the
|
||||
authenticated key's hash lands in ``metadata["user_api_key_hash"]``. Which of
|
||||
the two metadata dicts carries them depends on the endpoint, so both are read in
|
||||
the same precedence order everywhere.
|
||||
|
||||
The key hash is the trust boundary. A session id is caller-supplied in the end,
|
||||
so anything keyed by one has to be namespaced by the authenticated key as well;
|
||||
otherwise two callers reusing the same id read and write each other's entries.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
|
||||
_METADATA_KEYS = ("litellm_metadata", "metadata")
|
||||
|
||||
|
||||
def _metadata_dicts(request_kwargs: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(metadata for key in _METADATA_KEYS if isinstance(metadata := request_kwargs.get(key), Mapping))
|
||||
|
||||
|
||||
def _first_value(request_kwargs: Mapping[str, object], field: str) -> str | None:
|
||||
for metadata in _metadata_dicts(request_kwargs):
|
||||
value = metadata.get(field)
|
||||
if value is not None:
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def session_id_from_request(request_kwargs: Mapping[str, object]) -> str | None:
|
||||
"""The conversation this request continues, or ``None`` when nothing names one."""
|
||||
return _first_value(request_kwargs, "session_id")
|
||||
|
||||
|
||||
def user_api_key_hash_from_request(request_kwargs: Mapping[str, object]) -> str | None:
|
||||
"""The authenticated caller's key hash, or ``None`` outside the proxy."""
|
||||
return _first_value(request_kwargs, "user_api_key_hash")
|
||||
|
||||
|
||||
def session_scope(request_kwargs: Mapping[str, object], namespace: str, discriminator: str) -> str | None:
|
||||
"""A cache key for per-session state, or ``None`` when the request names no session.
|
||||
|
||||
``discriminator`` separates two features keyed off the same session, and the
|
||||
caller's key hash separates two callers who picked the same session id. Falling
|
||||
back to ``unscoped`` covers direct Router use, where there is no authenticated
|
||||
caller to scope by and no cross-caller collision to prevent.
|
||||
"""
|
||||
session_id = session_id_from_request(request_kwargs)
|
||||
if session_id is None:
|
||||
return None
|
||||
caller_scope = user_api_key_hash_from_request(request_kwargs) or "unscoped"
|
||||
return f"{discriminator}:v1:{namespace}:{caller_scope}:{session_id}"
|
||||
|
|
@ -842,6 +842,8 @@ class PreRoutingHookResponse(BaseModel):
|
|||
messages: Optional[List[Dict[str, Any]]]
|
||||
routing_decision: StandardLoggingRoutingDecision | None = None
|
||||
savings_baseline_model: Optional[str] = None
|
||||
previous_model: Optional[str] = None
|
||||
session_tracked: bool = False
|
||||
|
||||
|
||||
_PreRoutingStrategyT_co = TypeVar("_PreRoutingStrategyT_co", covariant=True)
|
||||
|
|
|
|||
|
|
@ -129,12 +129,27 @@ def _usage(fresh: int, cached: int, written: int, out: int) -> Usage:
|
|||
)
|
||||
|
||||
|
||||
def _savings(baseline: str, selected: str, usage: Usage) -> float:
|
||||
def _savings(
|
||||
baseline: str,
|
||||
selected: str,
|
||||
usage: Usage,
|
||||
previous: str | None = "claude-opus-5",
|
||||
tracked: bool = True,
|
||||
) -> float:
|
||||
"""Savings for a request, defaulting to a mid-conversation switch.
|
||||
|
||||
The default `previous` is some other model, because that is what makes a cold cache
|
||||
the switch's own cost. `previous=None` with `tracked=True` is a tracked session's
|
||||
first turn, where nothing was cached on either side; `tracked=False` is a request
|
||||
that named no session at all and therefore has no discriminator.
|
||||
"""
|
||||
return compute_autorouter_savings(
|
||||
baseline_model=baseline,
|
||||
selected_model=selected,
|
||||
selected_provider="anthropic",
|
||||
usage=usage,
|
||||
previous_model=previous,
|
||||
session_tracked=tracked,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -216,7 +231,7 @@ def test_multimodal_prompts_are_priced_on_the_baseline_too():
|
|||
total_tokens=21_000,
|
||||
prompt_tokens_details=details,
|
||||
)
|
||||
baseline = _baseline_usage(with_images)
|
||||
baseline = _baseline_usage(with_images, is_switch=True)
|
||||
|
||||
assert baseline.prompt_tokens_details.image_tokens == 4_000, "image tokens must survive into the baseline"
|
||||
|
||||
|
|
@ -243,7 +258,7 @@ def test_the_baseline_is_never_charged_a_cache_write():
|
|||
"cache_creation_token_details": {"ephemeral_1h_input_tokens": 20_000},
|
||||
},
|
||||
)
|
||||
baseline = _baseline_usage(long_cache)
|
||||
baseline = _baseline_usage(long_cache, is_switch=True)
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
priced, _ = generic_cost_per_token(model="claude-opus-5", usage=baseline, custom_llm_provider="anthropic")
|
||||
|
|
@ -299,6 +314,7 @@ def test_compute_savings_spend_carries_a_losing_switch_through():
|
|||
cache_read_input_tokens=0,
|
||||
baseline_model="claude-sonnet-5",
|
||||
usage_object=_cached_usage_object(),
|
||||
previous_model="claude-opus-5",
|
||||
)
|
||||
assert result.autorouter < 0
|
||||
|
||||
|
|
@ -370,3 +386,75 @@ def test_baseline_is_priced_under_its_own_provider():
|
|||
def test_unresolvable_baseline_fails_open_to_zero():
|
||||
usage = _usage(fresh=2000, cached=0, written=0, out=500)
|
||||
assert _savings("no-such-provider-xyz/no-such-model", "claude-haiku-4-5", usage) == 0.0
|
||||
|
||||
|
||||
def test_a_first_turn_is_the_rate_difference_not_a_switch_penalty():
|
||||
"""Nothing was cached anywhere on a conversation's first turn, so the baseline would
|
||||
have paid the same cache write. Charging it to the selected arm alone reported a
|
||||
fraction of the real saving; on this shape roughly 4% of it.
|
||||
"""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous=None)
|
||||
|
||||
opus = litellm.get_model_info("claude-opus-5", "anthropic")
|
||||
haiku = litellm.get_model_info("claude-haiku-4-5", "anthropic")
|
||||
both_write = (20_000 * opus["cache_creation_input_token_cost"] + 1_000 * opus["output_cost_per_token"]) - (
|
||||
20_000 * haiku["cache_creation_input_token_cost"] + 1_000 * haiku["output_cost_per_token"]
|
||||
)
|
||||
assert first_turn == pytest.approx(both_write)
|
||||
|
||||
charged_as_a_switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous="claude-sonnet-5")
|
||||
assert first_turn > charged_as_a_switch * 10, "the first turn must not be priced as a switch"
|
||||
|
||||
|
||||
def test_a_first_turn_that_saves_money_never_reports_a_loss():
|
||||
"""The write premium is fixed by prompt size while the saving grows with completion
|
||||
length, so charging the write to a first turn made short answers over a large cached
|
||||
prompt read as losses on requests that genuinely saved. That is the shape most likely
|
||||
to be on the dashboard, and the sign has to be right.
|
||||
"""
|
||||
short_answer = _usage(fresh=0, cached=0, written=20_000, out=200)
|
||||
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, previous=None) > 0
|
||||
assert _savings("anthropic/claude-opus-5", "claude-haiku-4-5", short_answer, previous="claude-sonnet-5") < 0
|
||||
|
||||
|
||||
def test_staying_on_one_model_is_not_a_switch():
|
||||
"""A session served the same model again pays its write because the conversation
|
||||
grew, not because anything moved; the baseline would have paid it too."""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
stayed = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous="claude-haiku-4-5")
|
||||
brand_new = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous=None)
|
||||
assert stayed == pytest.approx(brand_new)
|
||||
|
||||
|
||||
def test_the_previous_model_is_resolved_not_string_compared():
|
||||
"""It is recorded as the router's own model-group name while the served model arrives
|
||||
normalized from the spend log, so raw equality reads one deployment as two and
|
||||
invents a switch penalty on a session that never moved."""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
qualified = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous="anthropic/claude-haiku-4-5")
|
||||
bare = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous="claude-haiku-4-5")
|
||||
assert qualified == pytest.approx(bare)
|
||||
|
||||
|
||||
def test_an_unresolvable_previous_model_stays_conservative():
|
||||
"""A name that resolves to nothing cannot prove a switch happened, and must not be
|
||||
treated as one; the conservative rule understates rather than inflates."""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
unknown = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous="no-such-provider-xyz/nope")
|
||||
brand_new = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous=None)
|
||||
assert unknown == pytest.approx(brand_new)
|
||||
|
||||
|
||||
def test_a_request_without_a_session_stays_conservative():
|
||||
"""No session id means nothing can say why the cache was cold, and a savings claim
|
||||
must not inflate on a guess. The untracked request is priced exactly as a switch,
|
||||
which under-claims a first turn rather than crediting one that never happened.
|
||||
"""
|
||||
usage = _usage(fresh=0, cached=0, written=20_000, out=1_000)
|
||||
untracked = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous=None, tracked=False)
|
||||
switch = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous="claude-sonnet-5")
|
||||
tracked_first_turn = _savings("anthropic/claude-opus-5", "claude-haiku-4-5", usage, previous=None)
|
||||
|
||||
assert untracked == pytest.approx(switch), "an untracked request must be priced as a switch"
|
||||
assert untracked < tracked_first_turn, "and must never claim the first turn's larger saving"
|
||||
|
|
|
|||
|
|
@ -2437,7 +2437,7 @@ class TestSpendLogsPayload:
|
|||
"model": "gpt-4o",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "auto_router_previous_model": null, "auto_router_session_tracked": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.00022500000000000002,
|
||||
"total_tokens": 30,
|
||||
|
|
@ -2533,7 +2533,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "auto_router_previous_model": null, "auto_router_session_tracked": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
@ -2627,7 +2627,7 @@ class TestSpendLogsPayload:
|
|||
"model": "claude-4-sonnet-20250514",
|
||||
"user": "",
|
||||
"team_id": "",
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "auto_router_savings_baseline_model": null, "auto_router_previous_model": null, "auto_router_session_tracked": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
|
||||
"cache_key": "Cache OFF",
|
||||
"spend": 0.01383,
|
||||
"total_tokens": 2598,
|
||||
|
|
|
|||
|
|
@ -3012,6 +3012,22 @@ class TestSessionAffinity:
|
|||
assert reasoning.model == "o1-preview"
|
||||
assert simple.model == "gpt-4o-mini"
|
||||
|
||||
@staticmethod
|
||||
def _affinity_write(cache) -> dict:
|
||||
"""The kwargs of the session-affinity write.
|
||||
|
||||
Asserted by key rather than by call count: the router also records the model
|
||||
served to the session for the savings baseline, so a bare `assert_called_once`
|
||||
would couple these tests to an unrelated feature's cache traffic.
|
||||
"""
|
||||
writes = [
|
||||
call.kwargs
|
||||
for call in cache.async_set_cache.call_args_list
|
||||
if call.kwargs.get("key", "").startswith("complexity_router_session_affinity:")
|
||||
]
|
||||
assert len(writes) == 1, f"expected exactly one session-affinity write, got {writes}"
|
||||
return writes[0]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_respects_ttl_seconds(self, mock_router_instance, basic_config):
|
||||
cache = AsyncMock()
|
||||
|
|
@ -3029,8 +3045,7 @@ class TestSessionAffinity:
|
|||
await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=self._request_kwargs("session-1"), messages=self.SIMPLE_MESSAGE
|
||||
)
|
||||
cache.async_set_cache.assert_called_once()
|
||||
call_kwargs = cache.async_set_cache.call_args.kwargs
|
||||
call_kwargs = self._affinity_write(cache)
|
||||
assert call_kwargs["ttl"] == 120
|
||||
assert call_kwargs["value"] == "gpt-4o-mini"
|
||||
|
||||
|
|
@ -3054,8 +3069,7 @@ class TestSessionAffinity:
|
|||
model="test-model", request_kwargs=self._request_kwargs("session-1"), messages=self.SIMPLE_MESSAGE
|
||||
)
|
||||
assert result.model == "o1-preview"
|
||||
cache.async_set_cache.assert_called_once()
|
||||
call_kwargs = cache.async_set_cache.call_args.kwargs
|
||||
call_kwargs = self._affinity_write(cache)
|
||||
assert call_kwargs["value"] == "o1-preview"
|
||||
assert call_kwargs["ttl"] == 90
|
||||
|
||||
|
|
@ -4842,3 +4856,149 @@ class TestSavingsBaselineModel:
|
|||
returns = source.count("return PreRoutingHookResponse(")
|
||||
assert returns > 0
|
||||
assert source.count("savings_baseline_model=self.savings_baseline_model") == returns
|
||||
|
||||
|
||||
class TestPreviousModelForSession:
|
||||
"""What a session was served last, which is what tells a switch from a first turn."""
|
||||
|
||||
SIMPLE = [{"role": "user", "content": "Hello!"}]
|
||||
REASONING = [{"role": "user", "content": "Let's think step by step and reason through this carefully."}]
|
||||
|
||||
@staticmethod
|
||||
def _kwargs(session_id: str | None = "session-1", key_hash: str | None = None) -> Dict:
|
||||
metadata: Dict = {}
|
||||
if session_id is not None:
|
||||
metadata["session_id"] = session_id
|
||||
if key_hash is not None:
|
||||
metadata["user_api_key_hash"] = key_hash
|
||||
return {"metadata": metadata}
|
||||
|
||||
@staticmethod
|
||||
def _router(mock_router_instance, basic_config, **overrides) -> ComplexityRouter:
|
||||
return ComplexityRouter(
|
||||
model_name="test-router",
|
||||
litellm_router_instance=mock_router_instance,
|
||||
complexity_router_config={**basic_config, **overrides},
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_turn_reports_no_previous_model(self, mock_router_instance, basic_config):
|
||||
"""Nothing was cached anywhere yet, so both arms would have paid the write and
|
||||
the baseline must not be credited a read it never took."""
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
first = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=self._kwargs(), messages=self.SIMPLE
|
||||
)
|
||||
assert first.previous_model is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_later_turn_reports_what_the_session_was_served_before(self, mock_router_instance, basic_config):
|
||||
"""The discriminator: turn two knows turn one ran on a different model, which is
|
||||
what makes its cold cache the switch's own cost rather than a first write."""
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
request_kwargs = self._kwargs()
|
||||
first = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.REASONING
|
||||
)
|
||||
second = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE
|
||||
)
|
||||
assert first.model == "o1-preview"
|
||||
assert second.model == "gpt-4o-mini"
|
||||
assert second.previous_model == "o1-preview", "turn two must see turn one's model"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_staying_on_one_model_reports_that_same_model(self, mock_router_instance, basic_config):
|
||||
"""Equal is not a switch. Reporting the model unchanged lets the pricing treat
|
||||
the write as a first write rather than a switch penalty."""
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
request_kwargs = self._kwargs()
|
||||
await router.async_pre_routing_hook(model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE)
|
||||
second = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE
|
||||
)
|
||||
assert second.previous_model == "gpt-4o-mini"
|
||||
assert second.model == "gpt-4o-mini"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_request_without_a_session_never_touches_the_cache(self, mock_router_instance, basic_config):
|
||||
"""No session means no discriminator and nothing worth a round-trip; the pricing
|
||||
falls back to the conservative rule on its own."""
|
||||
cache = AsyncMock()
|
||||
cache.async_get_cache = AsyncMock(return_value=None)
|
||||
mock_router_instance.cache = cache
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs={"metadata": {}}, messages=self.SIMPLE
|
||||
)
|
||||
assert result.previous_model is None
|
||||
assert cache.async_get_cache.await_count == 0
|
||||
assert cache.async_set_cache.await_count == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_callers_sharing_a_session_id_do_not_see_each_others_models(
|
||||
self, mock_router_instance, basic_config
|
||||
):
|
||||
"""A session_id is caller-supplied, so without the authenticated key in the scope
|
||||
one tenant's model would be read as another's previous turn."""
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs=self._kwargs(key_hash="hash-a"),
|
||||
messages=self.REASONING,
|
||||
)
|
||||
other = await router.async_pre_routing_hook(
|
||||
model="test-model",
|
||||
request_kwargs=self._kwargs(key_hash="hash-b"),
|
||||
messages=self.SIMPLE,
|
||||
)
|
||||
assert other.previous_model is None, "the other caller's model must not leak across the key boundary"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_cache_failure_does_not_fail_the_request(self, mock_router_instance, basic_config):
|
||||
"""The counterfactual is a dashboard number; losing it must never cost a live call."""
|
||||
cache = AsyncMock()
|
||||
cache.async_get_cache = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
cache.async_set_cache = AsyncMock(side_effect=RuntimeError("redis down"))
|
||||
mock_router_instance.cache = cache
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
result = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=self._kwargs(), messages=self.SIMPLE
|
||||
)
|
||||
assert result is not None and result.model == "gpt-4o-mini"
|
||||
assert result.previous_model is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_previous_model_travels_on_every_pre_routing_response(self):
|
||||
"""A response without it silently falls back to charging the write on a first turn."""
|
||||
import inspect
|
||||
|
||||
from litellm.router_strategy.complexity_router import complexity_router as module
|
||||
|
||||
source = inspect.getsource(module.ComplexityRouter.async_pre_routing_hook) + inspect.getsource(
|
||||
module.ComplexityRouter._classify_and_route
|
||||
)
|
||||
constructions = source.split("return PreRoutingHookResponse(")[1:]
|
||||
assert len(constructions) > 0
|
||||
missing = [i for i, block in enumerate(constructions) if "previous_model=previous_model" not in block.split(")\n")[0]]
|
||||
assert not missing, f"pre-routing responses {missing} do not carry previous_model"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_it_is_recorded_independently_of_session_affinity(self, mock_router_instance, basic_config):
|
||||
"""`session_affinity` pins a model; this only observes one. Coupling them would
|
||||
make the savings number depend on an unrelated routing feature being enabled."""
|
||||
mock_router_instance.cache = DualCache()
|
||||
router = self._router(mock_router_instance, basic_config, session_affinity=False)
|
||||
request_kwargs = self._kwargs()
|
||||
first = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.REASONING
|
||||
)
|
||||
second = await router.async_pre_routing_hook(
|
||||
model="test-model", request_kwargs=request_kwargs, messages=self.SIMPLE
|
||||
)
|
||||
assert first.model != second.model, "affinity is off, so the model must be free to change"
|
||||
assert second.previous_model == first.model
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue