fix(router): give cost-based routing its own cache key so it stops overwriting latency samples (#40225)

* fix(router): give cost-based routing its own cache key so it stops overwriting latency samples

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(router): prefix the cost routing cache key so it cannot alias another group's latency key

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

litellm/router_strategy/lowest_cost.py

@@ -39,7 +39,7 @@ class LowestCostLoggingHandler(CustomLogger):
                 # ------------
                 """
                 {
-                    {model_group}_map: {
+                    cost_map:{model_group}: {
                         id: {
                             f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
                         }
@@ -50,7 +50,7 @@ class LowestCostLoggingHandler(CustomLogger):
                 current_hour: Final = datetime.now().strftime("%H")
                 current_minute: Final = datetime.now().strftime("%M")
                 precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
-                cost_key: Final = f"{model_group}_map"
+                cost_key: Final = f"cost_map:{model_group}"
 
                 total_tokens = 0
 
@@ -112,15 +112,14 @@ class LowestCostLoggingHandler(CustomLogger):
                 # ------------
                 """
                 {
-                    {model_group}_map: {
+                    cost_map:{model_group}: {
                         id: {
-                            "cost": [..]
                             f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
                         }
                     }
                 }
                 """
-                cost_key: Final = f"{model_group}_map"
+                cost_key: Final = f"cost_map:{model_group}"
 
                 current_date: Final = datetime.now().strftime("%Y-%m-%d")
                 current_hour: Final = datetime.now().strftime("%H")
@@ -176,7 +175,7 @@ class LowestCostLoggingHandler(CustomLogger):
         """
         Returns a deployment with the lowest cost
         """
-        cost_key: Final = f"{model_group}_map"
+        cost_key: Final = f"cost_map:{model_group}"
 
         request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {}
 

tests/test_litellm/router_strategy/test_lowest_cost.py

@@ -1,3 +1,4 @@
+import copy
 from datetime import datetime
 
 import pytest
@@ -7,6 +8,8 @@ from litellm.caching.caching import DualCache
 from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
 
 DEPLOYMENT_ID = "9876"
+COST_KEY = "cost_map:gpt-5.5-pool"
+LATENCY_KEYS = ("gpt-5.5-pool_map", "gpt-5.5-pool_cost_map")
 KWARGS = {
     "litellm_params": {
         "metadata": {"model_group": "gpt-5.5-pool"},
@@ -24,7 +27,7 @@ def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse:
 
 
 def _recorded_minute_counters(cache: DualCache) -> dict[str, int]:
-    cached = cache.get_cache(key="gpt-5.5-pool_map") or {}
+    cached = cache.get_cache(key=COST_KEY) or {}
     minute_buckets = cached.get(DEPLOYMENT_ID, {})
     assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}"
     return next(iter(minute_buckets.values()))
@@ -44,6 +47,47 @@ def test_log_success_event_counts_a_response_with_no_completion_tokens():
     assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"])
+async def test_log_success_event_keeps_cost_bookkeeping_out_of_the_latency_routing_entry(use_async: bool):
+    cache = DualCache()
+    latency_entry = {DEPLOYMENT_ID: {"latency": [0.5], "time_to_first_token": [0.1]}}
+    for latency_key in LATENCY_KEYS:
+        cache.set_cache(key=latency_key, value=copy.deepcopy(latency_entry))
+    handler = LowestCostLoggingHandler(router_cache=cache)
+    call_args = {
+        "kwargs": KWARGS,
+        "response_obj": _chat_response_with_no_completion_tokens(),
+        "start_time": datetime(2026, 1, 1, 12, 0, 0),
+        "end_time": datetime(2026, 1, 1, 12, 0, 2),
+    }
+
+    if use_async:
+        await handler.async_log_success_event(**call_args)
+    else:
+        handler.log_success_event(**call_args)
+
+    assert [cache.get_cache(key=latency_key) for latency_key in LATENCY_KEYS] == [latency_entry, latency_entry]
+    assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
+
+
+@pytest.mark.asyncio
+async def test_async_get_available_deployments_applies_rpm_limit_from_the_cost_entry():
+    cache = DualCache()
+    handler = LowestCostLoggingHandler(router_cache=cache)
+    precise_minute = datetime.now().strftime("%Y-%m-%d-%H-%M")
+    cache.set_cache(key=COST_KEY, value={DEPLOYMENT_ID: {precise_minute: {"tpm": 12, "rpm": 1}}})
+    healthy_deployments = [{"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {"model": "gpt-5.5", "rpm": 1}}]
+
+    picked = await handler.async_get_available_deployments(
+        model_group="gpt-5.5-pool",
+        healthy_deployments=healthy_deployments,
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert picked is None
+
+
 @pytest.mark.asyncio
 async def test_async_log_success_event_counts_a_response_with_no_completion_tokens():
     cache = DualCache()