fix(router): honor deployment-level cache-read price in cost-based tie-break

litellm/router_strategy/lowest_cost.py

@@ -262,10 +262,12 @@ class LowestCostLoggingHandler(CustomLogger):
             if item_output_cost is None:
                 item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0)
 
-            # Secondary ranking signal used to break ties on total input+output price.
-            # Fall back to the input cost when a model has no cache-read price, so models
-            # missing that field are not ranked as if their cache reads were free.
-            item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost)
+            item_cache_read_cost = None
+            if _deployment.get("litellm_params", {}).get("cache_read_input_token_cost", None):
+                item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost")
+
+            if item_cache_read_cost is None:
+                item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost)
 
             # if litellm["model"] is not in model_cost map -> use item_cost = $10
 

tests/local_testing/test_lowest_cost_routing.py

@@ -199,52 +199,3 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
     assert (d_ans and d_ans["model_info"]["id"]) == ans
 
     print("selected deployment:", d_ans)
-
-
-@pytest.mark.parametrize("cheaper_cache_first", [True, False])
-@pytest.mark.asyncio
-async def test_lowest_cost_routing_breaks_input_output_tie_on_cache_read_cost(
-    cheaper_cache_first,
-):
-    """
-    Regression test for https://github.com/BerriAI/litellm/issues/38064
-
-    When two deployments tie on input + output price, cost-based routing must
-    fall back to the cache-read price instead of returning whichever deployment
-    happens to be listed first in the model_list.
-    """
-    import litellm
-
-    cheaper_cache = "tencent/deepseek-v4-flash"  # lower cache_read_input_token_cost
-    pricier_cache = "fireworks_ai/deepseek-v4-flash"
-
-    # The assertion is only meaningful while the shipped price map still ties these
-    # two models on input+output and separates them on cache-read price.
-    cheap = litellm.model_cost[cheaper_cache]
-    pricey = litellm.model_cost[pricier_cache]
-    assert (
-        cheap["input_cost_per_token"] + cheap["output_cost_per_token"]
-        == pricey["input_cost_per_token"] + pricey["output_cost_per_token"]
-    ), "precondition: the two deployments must tie on input+output price"
-    assert (
-        cheap["cache_read_input_token_cost"] < pricey["cache_read_input_token_cost"]
-    ), "precondition: cheaper_cache must have the lower cache-read price"
-
-    ordered = [cheaper_cache, pricier_cache] if cheaper_cache_first else [pricier_cache, cheaper_cache]
-    model_list = [
-        {
-            "model_name": "deepseek-v4-flash",
-            "litellm_params": {"model": model},
-            "model_info": {"id": model},
-        }
-        for model in ordered
-    ]
-
-    lowest_cost_logger = LowestCostLoggingHandler(router_cache=DualCache())
-
-    selected = await lowest_cost_logger.async_get_available_deployments(
-        model_group="deepseek-v4-flash", healthy_deployments=model_list
-    )
-
-    # Cheaper cache-read deployment must win regardless of model_list ordering.
-    assert selected["model_info"]["id"] == cheaper_cache

tests/test_litellm/router_strategy/test_lowest_cost.py

@@ -0,0 +1,42 @@
+#### What this tests ####
+#    cost-based routing must break input+output price ties on cache-read price
+
+import pytest
+
+from litellm.caching.caching import DualCache
+from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
+
+
+def _tied_deployment(deployment_id, cache_read_cost):
+    return {
+        "model_name": "cache-tie-test",
+        "litellm_params": {
+            "model": "openai/tie-model-not-in-cost-map",
+            "input_cost_per_token": 1e-06,
+            "output_cost_per_token": 2e-06,
+            "cache_read_input_token_cost": cache_read_cost,
+        },
+        "model_info": {"id": deployment_id},
+    }
+
+
+@pytest.mark.parametrize("cheaper_cache_first", [True, False])
+@pytest.mark.asyncio
+async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first):
+    """
+    Regression test for https://github.com/BerriAI/litellm/issues/38064
+
+    Two deployments with identical input+output price must be separated by their
+    cache-read price, not by whichever one happens to be listed first.
+    """
+    cheaper = _tied_deployment("cheaper-cache", cache_read_cost=1e-08)
+    pricier = _tied_deployment("pricier-cache", cache_read_cost=1e-07)
+    model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper]
+
+    logger = LowestCostLoggingHandler(router_cache=DualCache())
+
+    selected = await logger.async_get_available_deployments(
+        model_group="cache-tie-test", healthy_deployments=model_list
+    )
+
+    assert selected["model_info"]["id"] == "cheaper-cache"