scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(router): avoid mutable-dict literal in cache-read tie-break lookup

Browse source
This commit is contained in:
LancyZhao 2026-08-24 19:45:52 +08:00
parent bdfb64cfdc
commit a168bab8a9
No known key found for this signature in database
2 changed files with 27 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/router_strategy/lowest_cost.py
View file

@ -262,12 +262,12 @@ class LowestCostLoggingHandler(CustomLogger):
if item_output_cost is None:
item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0)
item_cache_read_cost = None
if _deployment.get("litellm_params", {}).get("cache_read_input_token_cost", None):
item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost")
if item_cache_read_cost is None:
item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost)
deployment_params = _deployment.get("litellm_params")
item_cache_read_cost = (
deployment_params.get("cache_read_input_token_cost")
if deployment_params and deployment_params.get("cache_read_input_token_cost") is not None
else item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost)
)
# if litellm["model"] is not in model_cost map -> use item_cost = $10

37
tests/test_litellm/router_strategy/test_lowest_cost.py
View file

@ -7,19 +7,6 @@ from litellm.caching.caching import DualCache
from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
def _tied_deployment(deployment_id, cache_read_cost):
return {
"model_name": "cache-tie-test",
"litellm_params": {
"model": "openai/tie-model-not-in-cost-map",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"cache_read_input_token_cost": cache_read_cost,
},
"model_info": {"id": deployment_id},
}
@pytest.mark.parametrize("cheaper_cache_first", [True, False])
@pytest.mark.asyncio
async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first):
@ -27,10 +14,28 @@ async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_c
Regression test for https://github.com/BerriAI/litellm/issues/38064
Two deployments with identical input+output price must be separated by their
cache-read price, not by whichever one happens to be listed first.
cache-read price, not by whichever one happens to be listed first. The pricier
deployment omits a cache-read price to exercise the input-cost fallback.
"""
cheaper = _tied_deployment("cheaper-cache", cache_read_cost=1e-08)
pricier = _tied_deployment("pricier-cache", cache_read_cost=1e-07)
cheaper = {
"model_name": "cache-tie-test",
"litellm_params": {
"model": "openai/tie-model-not-in-cost-map",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
"cache_read_input_token_cost": 1e-08,
},
"model_info": {"id": "cheaper-cache"},
}
pricier = {
"model_name": "cache-tie-test",
"litellm_params": {
"model": "openai/tie-model-not-in-cost-map",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
},
"model_info": {"id": "pricier-cache"},
}
model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper]
logger = LowestCostLoggingHandler(router_cache=DualCache())