Merge 784f7b92a5 into 1df25e26cf

litellm/router_strategy/lowest_cost.py

@@ -265,6 +265,14 @@ class LowestCostLoggingHandler(CustomLogger):
             # if litellm["model"] is not in model_cost map -> use item_cost = $10
 
             item_cost = item_input_cost + item_output_cost
+            # deployment override first, mirroring input/output above, then the cost map. An
+            # absent or explicitly null price falls back to the input price rather than 0, so a
+            # model without one is never ranked as though its cache reads were free
+            item_cache_read_cost = _deployment.get("litellm_params", {}).get("cache_read_input_token_cost")
+            if item_cache_read_cost is None:
+                item_cache_read_cost = item_litellm_model_cost_map.get("cache_read_input_token_cost")
+            if item_cache_read_cost is None:
+                item_cache_read_cost = item_input_cost
 
             item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)
             item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
@@ -294,12 +302,12 @@ class LowestCostLoggingHandler(CustomLogger):
             ):  # if user passed in tpm / rpm in the model_list
                 continue
             else:
-                potential_deployments.append((_deployment, item_cost))
+                potential_deployments.append((_deployment, item_cost, item_cache_read_cost))
 
         if len(potential_deployments) == 0:
             return None
 
-        potential_deployments = sorted(potential_deployments, key=lambda x: x[1])
+        potential_deployments = sorted(potential_deployments, key=lambda x: (x[1], x[2]))
 
         selected_deployment: Final = potential_deployments[0][0]
         return selected_deployment

tests/test_litellm/router_strategy/test_lowest_cost.py

@@ -0,0 +1,122 @@
+#### What this tests ####
+#    cost-based-routing scored deployments on input + output price alone, so two
+#    deployments whose input and output prices tie were ordered only by their position
+#    in model_list. cache_read_input_token_cost can differ 10x across such a tie, which
+#    is the price that dominates cache-heavy traffic. Issue #38064.
+
+import pytest
+
+import litellm
+from litellm.caching.caching import DualCache
+from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
+
+MODEL_GROUP = "test-tied-price-model"
+CHEAP_CACHE = "cheap-cache-read"
+PRICEY_CACHE = "pricey-cache-read"
+
+
+def _deployments(order):
+    return [
+        {
+            "model_name": MODEL_GROUP,
+            "litellm_params": {"model": f"openai/{name}"},
+            "model_info": {"id": name},
+        }
+        for name in order
+    ]
+
+
+@pytest.fixture
+def tied_prices():
+    """Two models with identical input and output prices, cache-read differing 10x."""
+    for name, cache_read in ((PRICEY_CACHE, 2.8e-08), (CHEAP_CACHE, 2.8e-09)):
+        litellm.register_model(
+            {
+                f"openai/{name}": {
+                    "litellm_provider": "openai",
+                    "mode": "chat",
+                    "input_cost_per_token": 1.4e-07,
+                    "output_cost_per_token": 2.8e-07,
+                    "cache_read_input_token_cost": cache_read,
+                }
+            }
+        )
+    yield
+    for name in (PRICEY_CACHE, CHEAP_CACHE):
+        litellm.model_cost.pop(f"openai/{name}", None)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "order",
+    [[PRICEY_CACHE, CHEAP_CACHE], [CHEAP_CACHE, PRICEY_CACHE]],
+    ids=["pricey-listed-first", "cheap-listed-first"],
+)
+async def test_tied_input_output_price_breaks_on_cache_read_not_list_order(order, tied_prices):
+    handler = LowestCostLoggingHandler(router_cache=DualCache())
+    healthy = _deployments(order)
+
+    selected = await handler.async_get_available_deployments(model_group=MODEL_GROUP, healthy_deployments=healthy)
+
+    assert selected["model_info"]["id"] == CHEAP_CACHE
+
+
+@pytest.fixture
+def tied_prices_no_cache_entry():
+    """Same tie, but the cost map carries an explicit null cache-read price."""
+    for name in (PRICEY_CACHE, CHEAP_CACHE):
+        litellm.register_model(
+            {
+                f"openai/{name}": {
+                    "litellm_provider": "openai",
+                    "mode": "chat",
+                    "input_cost_per_token": 1.4e-07,
+                    "output_cost_per_token": 2.8e-07,
+                }
+            }
+        )
+        litellm.model_cost[f"openai/{name}"]["cache_read_input_token_cost"] = None
+    yield
+    for name in (PRICEY_CACHE, CHEAP_CACHE):
+        litellm.model_cost.pop(f"openai/{name}", None)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "order",
+    [[PRICEY_CACHE, CHEAP_CACHE], [CHEAP_CACHE, PRICEY_CACHE]],
+    ids=["pricey-listed-first", "cheap-listed-first"],
+)
+async def test_deployment_level_cache_read_override_breaks_the_tie(order, tied_prices):
+    """A per-deployment cache-read price must win over the shared model-map entry, the same way
+    input and output prices already honour litellm_params."""
+    handler = LowestCostLoggingHandler(router_cache=DualCache())
+    overrides = {PRICEY_CACHE: 9e-07, CHEAP_CACHE: 1e-09}
+    healthy = [
+        {
+            "model_name": MODEL_GROUP,
+            "litellm_params": {
+                "model": f"openai/{PRICEY_CACHE}",
+                "cache_read_input_token_cost": overrides[name],
+            },
+            "model_info": {"id": name},
+        }
+        for name in order
+    ]
+
+    selected = await handler.async_get_available_deployments(model_group=MODEL_GROUP, healthy_deployments=healthy)
+
+    assert selected["model_info"]["id"] == CHEAP_CACHE
+
+
+@pytest.mark.asyncio
+async def test_null_cache_read_price_does_not_break_sorting(tied_prices_no_cache_entry):
+    """An explicitly null cache-read price means unset, not zero, so it must fall back to the
+    input price instead of putting None into the sort key and raising TypeError."""
+    handler = LowestCostLoggingHandler(router_cache=DualCache())
+
+    selected = await handler.async_get_available_deployments(
+        model_group=MODEL_GROUP, healthy_deployments=_deployments([PRICEY_CACHE, CHEAP_CACHE])
+    )
+
+    assert selected["model_info"]["id"] in (PRICEY_CACHE, CHEAP_CACHE)