scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

test(router): lock deployment-level cache-read price of 0 as a real price

Browse source 
a168bab rewrote the deployment-level lookup to drop two mutable-dict literals
and, in doing so, replaced a truthiness check with an explicit `is not None`.
That silently fixed a behavior bug the commit message never claimed: a
deployment configuring `cache_read_input_token_cost: 0` was read as unset, fell
through to the cost map and then to the input-cost fallback, and lost the tie to
a deployment whose cache reads are actually billed. Providers that do not charge
for cache reads make 0 a real configuration, not an edge case.

No test covered it, so nothing stops the next rewrite from going back to `or` /
truthiness. This adds the missing case: verified failing on bdfb64c's
implementation (both list orders) and passing on the current one.
This commit is contained in:
LancyZhao 2026-09-02 15:58:34 +08:00
parent a168bab8a9
commit e696ab1850
No known key found for this signature in database
1 changed files with 41 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
tests/test_litellm/router_strategy/test_lowest_cost.py
View file

@ -45,3 +45,44 @@ async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_c
)
assert selected["model_info"]["id"] == "cheaper-cache"
@pytest.mark.parametrize("free_cache_first", [True, False])
@pytest.mark.asyncio
async def test_cost_routing_honors_zero_deployment_cache_read_cost(free_cache_first):
"""
A deployment-level cache_read_input_token_cost of 0 is a price, not a missing value.
Truthiness checks treat it as unset and fall through to the cost map / input-cost
fallback, which ranks a deployment whose cache reads are free as the priciest one
in a tie. Providers that do not charge for cache reads make this a real config.
"""
free = {
"model_name": "cache-zero-test",
"litellm_params": {
"model": "openai/zero-model-not-in-cost-map",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"cache_read_input_token_cost": 0.0,
},
"model_info": {"id": "free-cache"},
}
paid = {
"model_name": "cache-zero-test",
"litellm_params": {
"model": "openai/zero-model-not-in-cost-map",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"cache_read_input_token_cost": 1e-08,
},
"model_info": {"id": "paid-cache"},
}
model_list = [free, paid] if free_cache_first else [paid, free]
logger = LowestCostLoggingHandler(router_cache=DualCache())
selected = await logger.async_get_available_deployments(
model_group="cache-zero-test", healthy_deployments=model_list
)
assert selected["model_info"]["id"] == "free-cache"