scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

fix(spend): compare auto-router targets by deployment identity (#40206)

Browse source 
Preserve deployment identity through savings calculation, with canonical model fallback only when either ID is absent. Cover negotiated rates, unchanged deployments, alias/base-model cache accounting and missing IDs.

Fixes #38811. Based on the deployment-identity approach proposed by @QuantumBreakz in #38834.

Co-authored-by: Claude Code <noreply@anthropic.com>
This commit is contained in:
tin-berri 2026-09-07 23:29:42 -07:00 • committed by GitHub
parent 9dbfb060bd
commit 1a6aa98230
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 108 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
litellm/proxy/spend_tracking/savings.py
View file

@ -299,6 +299,8 @@ def compute_autorouter_savings(
selected_info: ModelInfo | None = None,
baseline_info: ModelInfo | None = None,
cost_breakdown: Mapping[str, object] | None = None,
baseline_deployment_id: str | None = None,
selected_deployment_id: str | None = None,
) -> float:
"""Net dollars the router saved, or cost, by serving this request on ``selected_model``.
@ -334,11 +336,12 @@ def compute_autorouter_savings(
selected: Final = _resolve_model(selected_model, selected_provider)
if baseline is None or selected is None:
return 0.0
# Same model is only the same cost when it is also the same deployment. Two
# deployments of one model can carry different negotiated rates, and routing from
# the dear one to the cheap one is a real saving that short-circuiting on the model
# name alone reports as zero.
if baseline == selected:
same_target: Final = (
baseline_deployment_id == selected_deployment_id
if baseline_deployment_id and selected_deployment_id
else baseline == selected
)
if same_target:
return 0.0
basis: Final = _pricing_basis(cost_breakdown)
effective_baseline_info: Final = baseline_info if baseline_info is not None else _model_info(baseline)
@ -517,6 +520,8 @@ def autorouter_savings_for_request(
selected_info=_effective_model_info(router_instance, model_id, model or ""),
baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
cost_breakdown=cost_breakdown,
baseline_deployment_id=baseline_id,
selected_deployment_id=model_id,
)
classifier_cost: Final = classifier_cost_from_decision(decision)
return gross if classifier_cost is None else gross - classifier_cost

98
tests/test_litellm/proxy/spend_tracking/test_savings.py
View file

@ -1162,6 +1162,104 @@ def test_prompt_caching_prices_at_the_deployment_rate_not_the_public_one():
assert result.prompt_caching > at_public_rates.prompt_caching
@pytest.mark.parametrize(
"baseline_id, selected_id, selected_multiplier, billed_input, classifier_cost, expected",
[
("baseline", "selected", 0.1, None, 0.0, 0.0135),
("baseline", "selected", 2.0, None, 0.0, -0.015),
("baseline", "selected", 1.0, None, 0.0, 0.0),
("baseline", "selected", 0.1, 0.004, 0.001, 0.01),
("baseline", "baseline", 0.1, 0.004, 0.001, -0.001),
(None, "selected", 0.1, None, 0.0, 0.0),
("baseline", None, 0.1, None, 0.0, 0.0),
(None, None, 0.1, None, 0.0, 0.0),
("", "selected", 0.1, None, 0.0, 0.0),
("baseline", "", 0.1, None, 0.0, 0.0),
],
)
def test_autorouter_savings_distinguishes_priced_deployments(
baseline_id: str | None,
selected_id: str | None,
selected_multiplier: float,
billed_input: float | None,
classifier_cost: float,
expected: float,
) -> None:
router: Final = Router(
model_list=[
{
"model_name": name,
"litellm_params": {
"model": "anthropic/claude-opus-5",
"api_key": "test-key",
"input_cost_per_token": 1e-5 * multiplier,
"output_cost_per_token": 5e-5 * multiplier,
},
"model_info": {"id": name},
}
for name, multiplier in (("baseline", 1.0), ("selected", selected_multiplier))
]
)
result: Final = compute_savings_spend(
model="claude-opus-5",
custom_llm_provider="anthropic",
compression_saved_tokens=0,
gateway_injected_cache=False,
model_id=selected_id,
llm_router=lambda: router,
routing_decision={
"savings_baseline_model": "anthropic/claude-opus-5",
"savings_baseline_deployment_id": baseline_id,
"conversation_continuing": False,
"classifier_cost": classifier_cost,
},
usage_object={"prompt_tokens": 1000, "completion_tokens": 100, "total_tokens": 1100},
cost_breakdown=None if billed_input is None else {"input_cost": billed_input, "output_cost": 0.0},
)
assert result.autorouter == pytest.approx(expected)
@pytest.mark.parametrize("selected_model", ["azure/contract-deployment", "contract-deployment"])
def test_autorouter_savings_recognizes_one_deployment_under_its_base_model(selected_model: str) -> None:
router: Final = Router(
model_list=[
{
"model_name": "contract",
"litellm_params": {
"model": "azure/contract-deployment",
"api_key": "test-key",
"api_base": "https://example.openai.azure.com",
"input_cost_per_token": 0.0001,
"output_cost_per_token": 0.0002,
"cache_read_input_token_cost": 0.00001,
},
"model_info": {"id": "contract", "base_model": "azure/gpt-5.5"},
}
]
)
result: Final = compute_savings_spend(
model=selected_model,
custom_llm_provider="azure",
compression_saved_tokens=0,
gateway_injected_cache=False,
model_id="contract",
llm_router=lambda: router,
routing_decision={
"savings_baseline_model": "azure/gpt-5.5",
"savings_baseline_deployment_id": "contract",
"conversation_continuing": True,
},
usage_object={
"prompt_tokens": 21000,
"completion_tokens": 100,
"total_tokens": 21100,
"prompt_tokens_details": {"text_tokens": 1000, "cached_tokens": 0, "cache_creation_tokens": 20000},
},
cost_breakdown={"input_cost": 2.1, "output_cost": 0.02},
)
assert result.autorouter == 0.0
def test_a_recorded_baseline_deployment_prices_at_its_configured_rate():
"""A hardest-tier deployment with a negotiated rate is what the traffic would
really have cost; pricing its model publicly misstates the saving."""