From c55248d113800f32283532cb811a2509b27ba94a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:07:29 -0700 Subject: [PATCH 1/2] fix(router): treat a routing entry with no latency samples as zero latency Latency-based routing averaged a deployment's cached samples with total / len(samples) and raised ZeroDivisionError once an entry held none, which the proxy answered as a 500 for every later request on that model group. Cost-based routing writes the same {model_group}_map entry with minute counters only, so a group used by both strategies hit this on every latency-routed request. A deployment with no samples now counts as 0 latency, the same as one the router has never seen Resolves LIT-7053 --- litellm/router_strategy/lowest_latency.py | 23 +++++++------- .../router_strategy/test_lowest_latency.py | 30 +++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index a1b67eaeaf9..4a1fb4f325a 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -1,6 +1,7 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) import random +from collections.abc import Sequence from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, Final @@ -25,6 +26,12 @@ class RoutingArgs(LiteLLMPydanticObjectBase): max_latency_list_size: int = 10 +def _average_latency(samples: Sequence[float | int]) -> float: + if not samples: + return 0.0 + return sum(sample for sample in samples if isinstance(sample, float)) / len(samples) + + class LowestLatencyLoggingHandler(CustomLogger): test_flag: bool = False logged_success: int = 0 @@ -431,23 +438,13 @@ class LowestLatencyLoggingHandler(CustomLogger): item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) # get average latency or average ttft (depending on streaming/non-streaming) - total: float = 0.0 use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 ) - if use_ttft: - for _call_latency in item_ttft_latency: - if isinstance(_call_latency, float): - total += _call_latency - item_latency = total / len(item_ttft_latency) - else: - for _call_latency in item_latency: - if isinstance(_call_latency, float): - total += _call_latency - item_latency = total / len(item_latency) + average_latency = _average_latency(item_ttft_latency if use_ttft else item_latency) # -------------- # # Debugging Logic @@ -456,7 +453,7 @@ class LowestLatencyLoggingHandler(CustomLogger): # this helps a user to debug why the router picked a specfic deployment # _deployment_api_base = _deployment.get("litellm_params", {}).get("api_base", "") if _deployment_api_base is not None: - _latency_per_deployment[_deployment_api_base] = item_latency + _latency_per_deployment[_deployment_api_base] = average_latency # -------------- # # End of Debugging Logic # -------------- # @@ -466,7 +463,7 @@ class LowestLatencyLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, item_latency)) + potential_deployments.append((_deployment, average_latency)) if len(potential_deployments) == 0: return None diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 6701f4a7aa2..251b760a8d8 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -163,3 +163,33 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): assert latencies and latencies[-1] == pytest.approx(2.0) assert not isinstance(latencies[-1], timedelta) json.dumps({"latency": latencies}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cached_entry", + [{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}], + ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"], +) +async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry): + """A cached entry with no latency samples (cost-based routing shares the group's map key and writes + minute buckets only) must count as 0 latency, like an unseen deployment, instead of dividing by zero.""" + cache = DualCache() + handler = LowestLatencyLoggingHandler(router_cache=cache) + cache.set_cache( + key="gemini-embedding-001_map", + value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}}, + ) + healthy_deployments = [ + {"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}}, + {"model_info": {"id": "slower"}, "litellm_params": {}}, + ] + + picked = await handler.async_get_available_deployments( + model_group="gemini-embedding-001", + healthy_deployments=healthy_deployments, + request_kwargs={"stream": False, "metadata": {}}, + ) + + assert picked is not None + assert picked["model_info"]["id"] == DEPLOYMENT_ID From be9a2ea7c97f945dc961c357b80f8b44f30c7746 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:20:20 -0700 Subject: [PATCH 2/2] fix(router): average every latency sample and drop the cost handler's dead division _average_latency skipped integer samples in the sum while counting them in the denominator, which contradicted its own Sequence[float | int] signature; it now averages every sample. Both success loggers in cost-based routing computed response_ms / completion_tokens and threw the result away, so a chat response with zero completion tokens raised ZeroDivisionError inside the handler. The proxy swallows and logs it, but the handler then skips that request's tpm and rpm update, so cost-based routing undercounts the deployment's usage. The QA run for the latency fix hit it on real gpt-5.5 traffic through /v1/chat/completions and /v1/messages --- litellm/router_strategy/lowest_cost.py | 11 +--- litellm/router_strategy/lowest_latency.py | 4 +- .../router_strategy/test_lowest_cost.py | 59 +++++++++++++++++++ .../router_strategy/test_lowest_latency.py | 2 - 4 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/router_strategy/test_lowest_cost.py diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..6eb4d86d280 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,6 +1,6 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) -from datetime import datetime, timedelta +from datetime import datetime from typing import Final import litellm @@ -52,16 +52,12 @@ class LowestCostLoggingHandler(CustomLogger): precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" cost_key: Final = f"{model_group}_map" - response_ms: Final[timedelta] = end_time - start_time - total_tokens = 0 if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) if _usage is not None and isinstance(_usage, litellm.Usage): - completion_tokens: Final = _usage.completion_tokens total_tokens = _usage.total_tokens - float(response_ms.total_seconds() / completion_tokens) # ------------ # Update usage @@ -131,18 +127,13 @@ class LowestCostLoggingHandler(CustomLogger): current_minute: Final = datetime.now().strftime("%M") precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" - response_ms: Final[timedelta] = end_time - start_time - total_tokens = 0 if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) if _usage is not None and isinstance(_usage, litellm.Usage): - completion_tokens: Final = _usage.completion_tokens total_tokens = _usage.total_tokens - float(response_ms.total_seconds() / completion_tokens) - # ------------ # Update usage # ------------ diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 4a1fb4f325a..bb6877a032b 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -26,10 +26,10 @@ class RoutingArgs(LiteLLMPydanticObjectBase): max_latency_list_size: int = 10 -def _average_latency(samples: Sequence[float | int]) -> float: +def _average_latency(samples: Sequence[float]) -> float: if not samples: return 0.0 - return sum(sample for sample in samples if isinstance(sample, float)) / len(samples) + return sum(samples) / len(samples) class LowestLatencyLoggingHandler(CustomLogger): diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..108053dddd9 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,59 @@ +from datetime import datetime + +import pytest + +import litellm +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + +DEPLOYMENT_ID = "9876" +KWARGS = { + "litellm_params": { + "metadata": {"model_group": "gpt-5.5-pool"}, + "model_info": {"id": DEPLOYMENT_ID}, + } +} + + +def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse: + return litellm.ModelResponse( + model="gpt-5.5", + choices=[{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}], + usage=litellm.Usage(prompt_tokens=12, completion_tokens=0, total_tokens=12), + ) + + +def _recorded_minute_counters(cache: DualCache) -> dict[str, int]: + cached = cache.get_cache(key="gpt-5.5-pool_map") or {} + minute_buckets = cached.get(DEPLOYMENT_ID, {}) + assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}" + return next(iter(minute_buckets.values())) + + +def test_log_success_event_counts_a_response_with_no_completion_tokens(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + + handler.log_success_event( + kwargs=KWARGS, + response_obj=_chat_response_with_no_completion_tokens(), + start_time=datetime(2026, 1, 1, 12, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 2), + ) + + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} + + +@pytest.mark.asyncio +async def test_async_log_success_event_counts_a_response_with_no_completion_tokens(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + + await handler.async_log_success_event( + kwargs=KWARGS, + response_obj=_chat_response_with_no_completion_tokens(), + start_time=datetime(2026, 1, 1, 12, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 2), + ) + + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 251b760a8d8..eb02459be68 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -172,8 +172,6 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"], ) async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry): - """A cached entry with no latency samples (cost-based routing shares the group's map key and writes - minute buckets only) must count as 0 latency, like an unseen deployment, instead of dividing by zero.""" cache = DualCache() handler = LowestLatencyLoggingHandler(router_cache=cache) cache.set_cache(