From 8dd25b9e661a7cc4ef76e72f693d33e1256889c4 Mon Sep 17 00:00:00 2001 From: LancyZhao <207653224+LancyZhao@users.noreply.github.com> Date: Mon, 24 Aug 2026 17:16:04 +0800 Subject: [PATCH 1/4] fix(router): break cost-based-routing input+output ties on cache-read price --- litellm/router_strategy/lowest_cost.py | 9 +++- .../local_testing/test_lowest_cost_routing.py | 49 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..cdcccf91921 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -262,6 +262,11 @@ class LowestCostLoggingHandler(CustomLogger): if item_output_cost is None: item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) + # Secondary ranking signal used to break ties on total input+output price. + # Fall back to the input cost when a model has no cache-read price, so models + # missing that field are not ranked as if their cache reads were free. + item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + # if litellm["model"] is not in model_cost map -> use item_cost = $10 item_cost = item_input_cost + item_output_cost @@ -294,12 +299,12 @@ class LowestCostLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, item_cost)) + potential_deployments.append((_deployment, item_cost, item_cache_read_cost)) if len(potential_deployments) == 0: return None - potential_deployments = sorted(potential_deployments, key=lambda x: x[1]) + potential_deployments = sorted(potential_deployments, key=lambda x: (x[1], x[2])) selected_deployment: Final = potential_deployments[0][0] return selected_deployment diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index 5bf3a3ee98b..4ad33f8db42 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -199,3 +199,52 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm): assert (d_ans and d_ans["model_info"]["id"]) == ans print("selected deployment:", d_ans) + + +@pytest.mark.parametrize("cheaper_cache_first", [True, False]) +@pytest.mark.asyncio +async def test_lowest_cost_routing_breaks_input_output_tie_on_cache_read_cost( + cheaper_cache_first, +): + """ + Regression test for https://github.com/BerriAI/litellm/issues/38064 + + When two deployments tie on input + output price, cost-based routing must + fall back to the cache-read price instead of returning whichever deployment + happens to be listed first in the model_list. + """ + import litellm + + cheaper_cache = "tencent/deepseek-v4-flash" # lower cache_read_input_token_cost + pricier_cache = "fireworks_ai/deepseek-v4-flash" + + # The assertion is only meaningful while the shipped price map still ties these + # two models on input+output and separates them on cache-read price. + cheap = litellm.model_cost[cheaper_cache] + pricey = litellm.model_cost[pricier_cache] + assert ( + cheap["input_cost_per_token"] + cheap["output_cost_per_token"] + == pricey["input_cost_per_token"] + pricey["output_cost_per_token"] + ), "precondition: the two deployments must tie on input+output price" + assert ( + cheap["cache_read_input_token_cost"] < pricey["cache_read_input_token_cost"] + ), "precondition: cheaper_cache must have the lower cache-read price" + + ordered = [cheaper_cache, pricier_cache] if cheaper_cache_first else [pricier_cache, cheaper_cache] + model_list = [ + { + "model_name": "deepseek-v4-flash", + "litellm_params": {"model": model}, + "model_info": {"id": model}, + } + for model in ordered + ] + + lowest_cost_logger = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await lowest_cost_logger.async_get_available_deployments( + model_group="deepseek-v4-flash", healthy_deployments=model_list + ) + + # Cheaper cache-read deployment must win regardless of model_list ordering. + assert selected["model_info"]["id"] == cheaper_cache From bdfb64cfdc7b697212a0859f923b8d8403b8c20b Mon Sep 17 00:00:00 2001 From: LancyZhao <207653224+LancyZhao@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:15:25 +0800 Subject: [PATCH 2/4] fix(router): honor deployment-level cache-read price in cost-based tie-break --- litellm/router_strategy/lowest_cost.py | 10 ++-- .../local_testing/test_lowest_cost_routing.py | 49 ------------------- .../router_strategy/test_lowest_cost.py | 42 ++++++++++++++++ 3 files changed, 48 insertions(+), 53 deletions(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_cost.py diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index cdcccf91921..8889180a994 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -262,10 +262,12 @@ class LowestCostLoggingHandler(CustomLogger): if item_output_cost is None: item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) - # Secondary ranking signal used to break ties on total input+output price. - # Fall back to the input cost when a model has no cache-read price, so models - # missing that field are not ranked as if their cache reads were free. - item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + item_cache_read_cost = None + if _deployment.get("litellm_params", {}).get("cache_read_input_token_cost", None): + item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost") + + if item_cache_read_cost is None: + item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) # if litellm["model"] is not in model_cost map -> use item_cost = $10 diff --git a/tests/local_testing/test_lowest_cost_routing.py b/tests/local_testing/test_lowest_cost_routing.py index 4ad33f8db42..5bf3a3ee98b 100644 --- a/tests/local_testing/test_lowest_cost_routing.py +++ b/tests/local_testing/test_lowest_cost_routing.py @@ -199,52 +199,3 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm): assert (d_ans and d_ans["model_info"]["id"]) == ans print("selected deployment:", d_ans) - - -@pytest.mark.parametrize("cheaper_cache_first", [True, False]) -@pytest.mark.asyncio -async def test_lowest_cost_routing_breaks_input_output_tie_on_cache_read_cost( - cheaper_cache_first, -): - """ - Regression test for https://github.com/BerriAI/litellm/issues/38064 - - When two deployments tie on input + output price, cost-based routing must - fall back to the cache-read price instead of returning whichever deployment - happens to be listed first in the model_list. - """ - import litellm - - cheaper_cache = "tencent/deepseek-v4-flash" # lower cache_read_input_token_cost - pricier_cache = "fireworks_ai/deepseek-v4-flash" - - # The assertion is only meaningful while the shipped price map still ties these - # two models on input+output and separates them on cache-read price. - cheap = litellm.model_cost[cheaper_cache] - pricey = litellm.model_cost[pricier_cache] - assert ( - cheap["input_cost_per_token"] + cheap["output_cost_per_token"] - == pricey["input_cost_per_token"] + pricey["output_cost_per_token"] - ), "precondition: the two deployments must tie on input+output price" - assert ( - cheap["cache_read_input_token_cost"] < pricey["cache_read_input_token_cost"] - ), "precondition: cheaper_cache must have the lower cache-read price" - - ordered = [cheaper_cache, pricier_cache] if cheaper_cache_first else [pricier_cache, cheaper_cache] - model_list = [ - { - "model_name": "deepseek-v4-flash", - "litellm_params": {"model": model}, - "model_info": {"id": model}, - } - for model in ordered - ] - - lowest_cost_logger = LowestCostLoggingHandler(router_cache=DualCache()) - - selected = await lowest_cost_logger.async_get_available_deployments( - model_group="deepseek-v4-flash", healthy_deployments=model_list - ) - - # Cheaper cache-read deployment must win regardless of model_list ordering. - assert selected["model_info"]["id"] == cheaper_cache diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..401466a557d --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,42 @@ +#### What this tests #### +# cost-based routing must break input+output price ties on cache-read price + +import pytest + +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + + +def _tied_deployment(deployment_id, cache_read_cost): + return { + "model_name": "cache-tie-test", + "litellm_params": { + "model": "openai/tie-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": cache_read_cost, + }, + "model_info": {"id": deployment_id}, + } + + +@pytest.mark.parametrize("cheaper_cache_first", [True, False]) +@pytest.mark.asyncio +async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first): + """ + Regression test for https://github.com/BerriAI/litellm/issues/38064 + + Two deployments with identical input+output price must be separated by their + cache-read price, not by whichever one happens to be listed first. + """ + cheaper = _tied_deployment("cheaper-cache", cache_read_cost=1e-08) + pricier = _tied_deployment("pricier-cache", cache_read_cost=1e-07) + model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper] + + logger = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await logger.async_get_available_deployments( + model_group="cache-tie-test", healthy_deployments=model_list + ) + + assert selected["model_info"]["id"] == "cheaper-cache" From a168bab8a95c6be82bb34d5ab0b26b082d23512e Mon Sep 17 00:00:00 2001 From: LancyZhao <207653224+LancyZhao@users.noreply.github.com> Date: Mon, 24 Aug 2026 19:45:52 +0800 Subject: [PATCH 3/4] fix(router): avoid mutable-dict literal in cache-read tie-break lookup --- litellm/router_strategy/lowest_cost.py | 12 +++--- .../router_strategy/test_lowest_cost.py | 37 +++++++++++-------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 8889180a994..a52c6d468ca 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -262,12 +262,12 @@ class LowestCostLoggingHandler(CustomLogger): if item_output_cost is None: item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) - item_cache_read_cost = None - if _deployment.get("litellm_params", {}).get("cache_read_input_token_cost", None): - item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost") - - if item_cache_read_cost is None: - item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + deployment_params = _deployment.get("litellm_params") + item_cache_read_cost = ( + deployment_params.get("cache_read_input_token_cost") + if deployment_params and deployment_params.get("cache_read_input_token_cost") is not None + else item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost) + ) # if litellm["model"] is not in model_cost map -> use item_cost = $10 diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py index 401466a557d..309780f217d 100644 --- a/tests/test_litellm/router_strategy/test_lowest_cost.py +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -7,19 +7,6 @@ from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler -def _tied_deployment(deployment_id, cache_read_cost): - return { - "model_name": "cache-tie-test", - "litellm_params": { - "model": "openai/tie-model-not-in-cost-map", - "input_cost_per_token": 1e-06, - "output_cost_per_token": 2e-06, - "cache_read_input_token_cost": cache_read_cost, - }, - "model_info": {"id": deployment_id}, - } - - @pytest.mark.parametrize("cheaper_cache_first", [True, False]) @pytest.mark.asyncio async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first): @@ -27,10 +14,28 @@ async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_c Regression test for https://github.com/BerriAI/litellm/issues/38064 Two deployments with identical input+output price must be separated by their - cache-read price, not by whichever one happens to be listed first. + cache-read price, not by whichever one happens to be listed first. The pricier + deployment omits a cache-read price to exercise the input-cost fallback. """ - cheaper = _tied_deployment("cheaper-cache", cache_read_cost=1e-08) - pricier = _tied_deployment("pricier-cache", cache_read_cost=1e-07) + cheaper = { + "model_name": "cache-tie-test", + "litellm_params": { + "model": "openai/tie-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-08, + }, + "model_info": {"id": "cheaper-cache"}, + } + pricier = { + "model_name": "cache-tie-test", + "litellm_params": { + "model": "openai/tie-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + }, + "model_info": {"id": "pricier-cache"}, + } model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper] logger = LowestCostLoggingHandler(router_cache=DualCache()) From e696ab1850c6703c4e77e91b4ac7d48a0ac9a5f8 Mon Sep 17 00:00:00 2001 From: LancyZhao <207653224+LancyZhao@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:58:34 +0800 Subject: [PATCH 4/4] test(router): lock deployment-level cache-read price of 0 as a real price a168bab rewrote the deployment-level lookup to drop two mutable-dict literals and, in doing so, replaced a truthiness check with an explicit `is not None`. That silently fixed a behavior bug the commit message never claimed: a deployment configuring `cache_read_input_token_cost: 0` was read as unset, fell through to the cost map and then to the input-cost fallback, and lost the tie to a deployment whose cache reads are actually billed. Providers that do not charge for cache reads make 0 a real configuration, not an edge case. No test covered it, so nothing stops the next rewrite from going back to `or` / truthiness. This adds the missing case: verified failing on bdfb64c's implementation (both list orders) and passing on the current one. --- .../router_strategy/test_lowest_cost.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py index 309780f217d..c95b16daeba 100644 --- a/tests/test_litellm/router_strategy/test_lowest_cost.py +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -45,3 +45,44 @@ async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_c ) assert selected["model_info"]["id"] == "cheaper-cache" + + +@pytest.mark.parametrize("free_cache_first", [True, False]) +@pytest.mark.asyncio +async def test_cost_routing_honors_zero_deployment_cache_read_cost(free_cache_first): + """ + A deployment-level cache_read_input_token_cost of 0 is a price, not a missing value. + + Truthiness checks treat it as unset and fall through to the cost map / input-cost + fallback, which ranks a deployment whose cache reads are free as the priciest one + in a tie. Providers that do not charge for cache reads make this a real config. + """ + free = { + "model_name": "cache-zero-test", + "litellm_params": { + "model": "openai/zero-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 0.0, + }, + "model_info": {"id": "free-cache"}, + } + paid = { + "model_name": "cache-zero-test", + "litellm_params": { + "model": "openai/zero-model-not-in-cost-map", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "cache_read_input_token_cost": 1e-08, + }, + "model_info": {"id": "paid-cache"}, + } + model_list = [free, paid] if free_cache_first else [paid, free] + + logger = LowestCostLoggingHandler(router_cache=DualCache()) + + selected = await logger.async_get_available_deployments( + model_group="cache-zero-test", healthy_deployments=model_list + ) + + assert selected["model_info"]["id"] == "free-cache"