scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-16 23:41:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(proxy): preserve Anthropic pricing modifiers in router savings

Browse source
This commit is contained in:
Tin Chi Lo 2026-09-15 18:02:05 -07:00
parent 878716f806
commit 5aa2adbacd
3 changed files with 79 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
litellm/llms/anthropic/cost_calculation.py
View file

@ -18,7 +18,9 @@ if TYPE_CHECKING:
import litellm
def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None) -> tuple[float, float]:
def cost_per_token(
model: str, usage: "Usage", service_tier: str | None = None, model_info: "ModelInfo | None" = None
) -> tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -27,6 +29,7 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
- usage: LiteLLM Usage block, containing anthropic caching information
- service_tier: the service tier the request was served at (e.g. "priority"),
read from the Anthropic response usage and used to select tier-specific pricing
- model_info: effective deployment prices, when they override public rates
Returns:
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
@ -36,16 +39,23 @@ def cost_per_token(model: str, usage: "Usage", service_tier: str | None = None)
usage=usage,
custom_llm_provider="anthropic",
service_tier=service_tier,
model_info=model_info,
)
# Apply provider_specific_entry multipliers for geo/speed routing
try:
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
provider_specific_entry: Final[dict] = model_info.get("provider_specific_entry") or {}
effective_info: Final = (
model_info
if model_info is not None
else litellm.get_model_info(model=model, custom_llm_provider="anthropic")
)
provider_specific_entry: Final = effective_info.get("provider_specific_entry")
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
geo_multiplier: Final = get_provider_specific_geo_multiplier(model_info=effective_info, usage=usage)
speed_multiplier: Final = (
provider_specific_entry.get("fast", 1.0) if getattr(usage, "speed", None) == "fast" else 1.0
provider_specific_entry.get("fast", 1.0)
if provider_specific_entry and getattr(usage, "speed", None) == "fast"
else 1.0
)
if speed_multiplier != 1.0:

62
litellm/proxy/spend_tracking/savings.py
View file

@ -171,15 +171,25 @@ def _cost_of_usage(
) -> float | None:
"""What ``usage`` costs on ``model``, or ``None`` when the model has no pricing."""
try:
prompt_cost, completion_cost = generic_cost_per_token(
model=model.model,
usage=usage,
custom_llm_provider=model.provider,
service_tier=basis.service_tier,
data_residency=basis.data_residency,
model_info=model_info,
vertex_location=basis.vertex_location,
)
if model.provider == "anthropic":
from litellm.llms.anthropic.cost_calculation import cost_per_token
prompt_cost, completion_cost = cost_per_token(
model=model.model,
usage=usage,
service_tier=basis.service_tier,
model_info=model_info,
)
else:
prompt_cost, completion_cost = generic_cost_per_token(
model=model.model,
usage=usage,
custom_llm_provider=model.provider,
service_tier=basis.service_tier,
data_residency=basis.data_residency,
model_info=model_info,
vertex_location=basis.vertex_location,
)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; degrade to zero savings
verbose_proxy_logger.debug(
"savings: cannot price usage for provider=%s model=%s (%s)", model.provider, model.model, e
@ -198,11 +208,6 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]:
return int(read), int(created)
_CACHE_SPLIT_FIELDS: Final = frozenset(
("cached_tokens", "cache_creation_tokens", "cache_write_tokens", "cache_creation_token_details", "text_tokens")
)
def _baseline_cache_rate_keys(baseline_info: ModelInfo | None) -> tuple[bool, bool]:
"""Whether the baseline model has a ``(cache read, cache write)`` rate of its own.
@ -274,19 +279,22 @@ def _baseline_usage(usage: Usage, conversation_continuing: bool, baseline_info:
(getattr(details, field, 0) or 0) for field in ("audio_tokens", "image_tokens", "video_tokens")
)
return Usage(
prompt_tokens=usage.prompt_tokens,
completion_tokens=usage.completion_tokens,
total_tokens=usage.total_tokens,
completion_tokens_details=usage.completion_tokens_details,
prompt_tokens_details=PromptTokensDetailsWrapper(
**details.model_dump(exclude=_CACHE_SPLIT_FIELDS),
cached_tokens=reads,
cache_creation_tokens=writes,
cache_write_tokens=writes,
cache_creation_token_details=details.cache_creation_token_details if writes else None,
# Whatever no longer sits in a cache bucket is plain input on the baseline.
text_tokens=max(usage.prompt_tokens - reads - writes - other_modalities, 0),
),
**{
**usage.model_dump(),
# Rebuild through Usage so private fallback counts agree with the public buckets.
"cache_read_input_tokens": reads,
"cache_creation_input_tokens": writes,
"prompt_tokens_details": PromptTokensDetailsWrapper(
**{
**details.model_dump(),
"cached_tokens": reads,
"cache_creation_tokens": writes,
"cache_write_tokens": writes,
"cache_creation_token_details": details.cache_creation_token_details if writes else None,
"text_tokens": max(usage.prompt_tokens - reads - writes - other_modalities, 0),
}
),
},
)

29
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -4,6 +4,7 @@ import pytest
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
from litellm.proxy.spend_tracking.savings import (
_baseline_usage,
_resolve_model,
@ -17,6 +18,34 @@ from litellm.types.utils import Usage
pytestmark = pytest.mark.usefixtures("local_model_cost_map")
@pytest.mark.parametrize("modifier", [{"speed": "fast"}, {"inference_geo": "us"}])
@pytest.mark.parametrize("continuing", [False, True])
def test_baseline_preserves_anthropic_pricing_fields(modifier: dict[str, str], continuing: bool) -> None:
usage: Final = _usage(1000, 0, 1000, 100).model_copy(update=modifier)
expected: Final = (_usage(1000, 1000, 0, 100) if continuing else usage).model_copy(update=modifier)
normalized: Final = _baseline_usage(usage, continuing)
cache_fields: Final = {"prompt_tokens_details", "cache_read_input_tokens", "cache_creation_input_tokens"}
assert normalized.model_dump(exclude=cache_fields) == usage.model_dump(exclude=cache_fields)
assert usage.prompt_tokens_details.cached_tokens == 0
selected_cost: Final = 0.013
assert compute_autorouter_savings(
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, conversation_continuing=continuing,
cost_breakdown={"input_cost": 0.01, "output_cost": 0.003},
) == pytest.approx(sum(anthropic_cost_per_token("claude-opus-5", expected)) - selected_cost)
def test_anthropic_baseline_keeps_negotiated_prices_with_provider_multiplier() -> None:
info: Final = {
**litellm.get_model_info("claude-opus-5", "anthropic"),
"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-6, "cache_read_input_token_cost": 3e-7,
}
usage: Final = _usage(1000, 1000, 0, 100).model_copy(update={"speed": "fast"})
assert compute_autorouter_savings(
"claude-opus-5", "claude-sonnet-5", "anthropic", usage, baseline_info=info,
cost_breakdown={"input_cost": 0.01, "output_cost": 0.003},
) == pytest.approx(0.0015 * 2 - 0.013)
def _anthropic_costs(model: str) -> tuple[float, float]:
info = litellm.get_model_info(model=model, custom_llm_provider="anthropic")
input_cost = info["input_cost_per_token"] or 0.0