From 9fffda4117536ff7996718ea8e9a1e1d4611063a Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:56:06 -0700 Subject: [PATCH] fix(router): give cost-based routing its own cache key so it stops overwriting latency samples (#40225) * fix(router): give cost-based routing its own cache key so it stops overwriting latency samples Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): prefix the cost routing cache key so it cannot alias another group's latency key Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/router_strategy/lowest_cost.py | 11 ++--- .../router_strategy/test_lowest_cost.py | 46 ++++++++++++++++++- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 6eb4d86d280..d271349914e 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -39,7 +39,7 @@ class LowestCostLoggingHandler(CustomLogger): # ------------ """ { - {model_group}_map: { + cost_map:{model_group}: { id: { f"{date:hour:minute}" : {"tpm": 34, "rpm": 3} } @@ -50,7 +50,7 @@ class LowestCostLoggingHandler(CustomLogger): current_hour: Final = datetime.now().strftime("%H") current_minute: Final = datetime.now().strftime("%M") precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" - cost_key: Final = f"{model_group}_map" + cost_key: Final = f"cost_map:{model_group}" total_tokens = 0 @@ -112,15 +112,14 @@ class LowestCostLoggingHandler(CustomLogger): # ------------ """ { - {model_group}_map: { + cost_map:{model_group}: { id: { - "cost": [..] f"{date:hour:minute}" : {"tpm": 34, "rpm": 3} } } } """ - cost_key: Final = f"{model_group}_map" + cost_key: Final = f"cost_map:{model_group}" current_date: Final = datetime.now().strftime("%Y-%m-%d") current_hour: Final = datetime.now().strftime("%H") @@ -176,7 +175,7 @@ class LowestCostLoggingHandler(CustomLogger): """ Returns a deployment with the lowest cost """ - cost_key: Final = f"{model_group}_map" + cost_key: Final = f"cost_map:{model_group}" request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {} diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py index 108053dddd9..ab3ef099410 100644 --- a/tests/test_litellm/router_strategy/test_lowest_cost.py +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -1,3 +1,4 @@ +import copy from datetime import datetime import pytest @@ -7,6 +8,8 @@ from litellm.caching.caching import DualCache from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler DEPLOYMENT_ID = "9876" +COST_KEY = "cost_map:gpt-5.5-pool" +LATENCY_KEYS = ("gpt-5.5-pool_map", "gpt-5.5-pool_cost_map") KWARGS = { "litellm_params": { "metadata": {"model_group": "gpt-5.5-pool"}, @@ -24,7 +27,7 @@ def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse: def _recorded_minute_counters(cache: DualCache) -> dict[str, int]: - cached = cache.get_cache(key="gpt-5.5-pool_map") or {} + cached = cache.get_cache(key=COST_KEY) or {} minute_buckets = cached.get(DEPLOYMENT_ID, {}) assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}" return next(iter(minute_buckets.values())) @@ -44,6 +47,47 @@ def test_log_success_event_counts_a_response_with_no_completion_tokens(): assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +async def test_log_success_event_keeps_cost_bookkeeping_out_of_the_latency_routing_entry(use_async: bool): + cache = DualCache() + latency_entry = {DEPLOYMENT_ID: {"latency": [0.5], "time_to_first_token": [0.1]}} + for latency_key in LATENCY_KEYS: + cache.set_cache(key=latency_key, value=copy.deepcopy(latency_entry)) + handler = LowestCostLoggingHandler(router_cache=cache) + call_args = { + "kwargs": KWARGS, + "response_obj": _chat_response_with_no_completion_tokens(), + "start_time": datetime(2026, 1, 1, 12, 0, 0), + "end_time": datetime(2026, 1, 1, 12, 0, 2), + } + + if use_async: + await handler.async_log_success_event(**call_args) + else: + handler.log_success_event(**call_args) + + assert [cache.get_cache(key=latency_key) for latency_key in LATENCY_KEYS] == [latency_entry, latency_entry] + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} + + +@pytest.mark.asyncio +async def test_async_get_available_deployments_applies_rpm_limit_from_the_cost_entry(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + precise_minute = datetime.now().strftime("%Y-%m-%d-%H-%M") + cache.set_cache(key=COST_KEY, value={DEPLOYMENT_ID: {precise_minute: {"tpm": 12, "rpm": 1}}}) + healthy_deployments = [{"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {"model": "gpt-5.5", "rpm": 1}}] + + picked = await handler.async_get_available_deployments( + model_group="gpt-5.5-pool", + healthy_deployments=healthy_deployments, + messages=[{"role": "user", "content": "hi"}], + ) + + assert picked is None + + @pytest.mark.asyncio async def test_async_log_success_event_counts_a_response_with_no_completion_tokens(): cache = DualCache()