Merge d1b358bef7 into aea5358c48

litellm/router_strategy/lowest_cost.py

@@ -262,6 +262,13 @@ class LowestCostLoggingHandler(CustomLogger):
             if item_output_cost is None:
                 item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0)
 
+            deployment_params = _deployment.get("litellm_params")
+            item_cache_read_cost = (
+                deployment_params.get("cache_read_input_token_cost")
+                if deployment_params and deployment_params.get("cache_read_input_token_cost") is not None
+                else item_litellm_model_cost_map.get("cache_read_input_token_cost", item_input_cost)
+            )
+
             # if litellm["model"] is not in model_cost map -> use item_cost = $10
 
             item_cost = item_input_cost + item_output_cost
@@ -294,12 +301,12 @@ class LowestCostLoggingHandler(CustomLogger):
             ):  # if user passed in tpm / rpm in the model_list
                 continue
             else:
-                potential_deployments.append((_deployment, item_cost))
+                potential_deployments.append((_deployment, item_cost, item_cache_read_cost))
 
         if len(potential_deployments) == 0:
             return None
 
-        potential_deployments = sorted(potential_deployments, key=lambda x: x[1])
+        potential_deployments = sorted(potential_deployments, key=lambda x: (x[1], x[2]))
 
         selected_deployment: Final = potential_deployments[0][0]
         return selected_deployment

tests/test_litellm/router_strategy/test_lowest_cost.py

@@ -0,0 +1,88 @@
+#### What this tests ####
+#    cost-based routing must break input+output price ties on cache-read price
+
+import pytest
+
+from litellm.caching.caching import DualCache
+from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
+
+
+@pytest.mark.parametrize("cheaper_cache_first", [True, False])
+@pytest.mark.asyncio
+async def test_cost_routing_breaks_input_output_tie_on_cache_read_cost(cheaper_cache_first):
+    """
+    Regression test for https://github.com/BerriAI/litellm/issues/38064
+
+    Two deployments with identical input+output price must be separated by their
+    cache-read price, not by whichever one happens to be listed first. The pricier
+    deployment omits a cache-read price to exercise the input-cost fallback.
+    """
+    cheaper = {
+        "model_name": "cache-tie-test",
+        "litellm_params": {
+            "model": "openai/tie-model-not-in-cost-map",
+            "input_cost_per_token": 1e-06,
+            "output_cost_per_token": 2e-06,
+            "cache_read_input_token_cost": 1e-08,
+        },
+        "model_info": {"id": "cheaper-cache"},
+    }
+    pricier = {
+        "model_name": "cache-tie-test",
+        "litellm_params": {
+            "model": "openai/tie-model-not-in-cost-map",
+            "input_cost_per_token": 1e-06,
+            "output_cost_per_token": 2e-06,
+        },
+        "model_info": {"id": "pricier-cache"},
+    }
+    model_list = [cheaper, pricier] if cheaper_cache_first else [pricier, cheaper]
+
+    logger = LowestCostLoggingHandler(router_cache=DualCache())
+
+    selected = await logger.async_get_available_deployments(
+        model_group="cache-tie-test", healthy_deployments=model_list
+    )
+
+    assert selected["model_info"]["id"] == "cheaper-cache"
+
+
+@pytest.mark.parametrize("free_cache_first", [True, False])
+@pytest.mark.asyncio
+async def test_cost_routing_honors_zero_deployment_cache_read_cost(free_cache_first):
+    """
+    A deployment-level cache_read_input_token_cost of 0 is a price, not a missing value.
+
+    Truthiness checks treat it as unset and fall through to the cost map / input-cost
+    fallback, which ranks a deployment whose cache reads are free as the priciest one
+    in a tie. Providers that do not charge for cache reads make this a real config.
+    """
+    free = {
+        "model_name": "cache-zero-test",
+        "litellm_params": {
+            "model": "openai/zero-model-not-in-cost-map",
+            "input_cost_per_token": 1e-06,
+            "output_cost_per_token": 2e-06,
+            "cache_read_input_token_cost": 0.0,
+        },
+        "model_info": {"id": "free-cache"},
+    }
+    paid = {
+        "model_name": "cache-zero-test",
+        "litellm_params": {
+            "model": "openai/zero-model-not-in-cost-map",
+            "input_cost_per_token": 1e-06,
+            "output_cost_per_token": 2e-06,
+            "cache_read_input_token_cost": 1e-08,
+        },
+        "model_info": {"id": "paid-cache"},
+    }
+    model_list = [free, paid] if free_cache_first else [paid, free]
+
+    logger = LowestCostLoggingHandler(router_cache=DualCache())
+
+    selected = await logger.async_get_available_deployments(
+        model_group="cache-zero-test", healthy_deployments=model_list
+    )
+
+    assert selected["model_info"]["id"] == "free-cache"