From bdfb64cfdc7b697212a0859f923b8d8403b8c20b Mon Sep 17 00:00:00 2001 From: LancyZhao <207653224+LancyZhao@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:15:25 +0800 Subject: [PATCH] fix(router): honor deployment-level cache-read price in cost-based tie-break --- litellm/router_strategy/lowest_cost.py | 10 ++-- .../local_testing/test_lowest_cost_routing.py | 49 ------------------- .../router_strategy/test_lowest_cost.py | 42 ++++++++++++++++ 3 files changed, 48 insertions(+), 53 deletions(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_cost.py diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index cdcccf91921..8889180a994 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -262,10 +262,12 @@ class LowestCostLoggingHandler(CustomLogger): if item_output_cost is None: item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) - # Secondary ranking signal used to break ties on total input+output price. - # Fall back to the input cost when a model has no cache-read price, so models - # missing that field are not ranked as if their cache reads were free. - item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + item_cache_read_cost = None + if _deployment.get("litellm_params", {}).get("cache_read_input_token_cost", None): + item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost") + + if item_cache_read_cost is None: + item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) # if litellm["model"] is not in model_cost map -> use item_cost = $10 diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index 4ad33f8db42..5bf3a3ee98b 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -199,52 +199,3 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm): assert (d_ans and d_ans["model_info"]["id"]) == ans print("selected deployment:", d_ans) - - -@pytest.mark.parametrize("cheaper_cache_first", [True, False]) -@pytest.mark.asyncio -async def test_lowest_cost_routing_breaks_input_output_tie_on_cache_read_cost( - cheaper_cache_first, -): - """ - Regression test for https://github.com/BerriAI/litellm/issues/38064 - - When two deployments tie on input + output price, cost-based routing must - fall back to the cache-read price instead of returning whichever deployment - happens to be listed first in the model_list. - """ - import litellm - - cheaper_cache = "tencent/deepseek-v4-flash" # lower cache_read_input_token_cost - pricier_cache = "fireworks_ai/deepseek-v4-flash" - - # The assertion is only meaningful while the shipped price map still ties these - # two models on input+output and separates them on cache-read price. - cheap = litellm.model_cost[cheaper_cache] - pricey = litellm.model_cost[pricier_cache] - assert ( - cheap["input_cost_per_token"] + cheap["output_cost_per_token"] - == pricey["input_cost_per_token"] + pricey["output_cost_per_token"] - ), "precondition: the two deployments must tie on input+output price" - assert ( - cheap["cache_read_input_token_cost"] < pricey["cache_read_input_token_cost"] - ), "precondition: cheaper_cache must have the lower cache-read price" - - ordered = [cheaper_cache, pricier_cache] if cheaper_cache_first else [pricier_cache, cheaper_cache] - model_list = [ - { - "model_name": "deepseek-v4-flash", - "litellm_params": {"model": model}, - "model_info": {"id": model}, - } - for model in ordered - ] - - lowest_cost_logger = LowestCostLoggingHandler(router_cache=DualCache()) - - selected = await lowest_cost_logger.async_get_available_deployments( - model_group="deepseek-v4-flash", healthy_deployments=model_list - ) - - # Cheaper cache-read deployment must win regardless of model_list ordering. - assert selected["model_info"]["id"] == cheaper_cache diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..401466a557d --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,42 @@ +#### What this tests #### +# cost-based routing must break input+output price ties on cache-read price + +import pytest + +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + + +def _tied_deployment(deployment_id, cache_read_cost): + return { + "model_name": "cache-tie-test", + "litellm_params": { + "model": "openai/tie-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": cache_read_cost, + }, + "model_info": {"id": deployment_id}, + } + + +@pytest.mark.parametrize("cheaper_cache_first", [True, False]) +@pytest.mark.asyncio +async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first): + """ + Regression test for https://github.com/BerriAI/litellm/issues/38064 + + Two deployments with identical input+output price must be separated by their + cache-read price, not by whichever one happens to be listed first. + """ + cheaper = _tied_deployment("cheaper-cache", cache_read_cost=1e-08) + pricier = _tied_deployment("pricier-cache", cache_read_cost=1e-07) + model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper] + + logger = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await logger.async_get_available_deployments( + model_group="cache-tie-test", healthy_deployments=model_list + ) + + assert selected["model_info"]["id"] == "cheaper-cache"