diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index b927df0c438..6eb4d86d280 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,6 +1,6 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) -from datetime import datetime, timedelta +from datetime import datetime from typing import Final import litellm @@ -52,16 +52,12 @@ class LowestCostLoggingHandler(CustomLogger): precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" cost_key: Final = f"{model_group}_map" - response_ms: Final[timedelta] = end_time - start_time - total_tokens = 0 if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) if _usage is not None and isinstance(_usage, litellm.Usage): - completion_tokens: Final = _usage.completion_tokens total_tokens = _usage.total_tokens - float(response_ms.total_seconds() / completion_tokens) # ------------ # Update usage @@ -131,18 +127,13 @@ class LowestCostLoggingHandler(CustomLogger): current_minute: Final = datetime.now().strftime("%M") precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}" - response_ms: Final[timedelta] = end_time - start_time - total_tokens = 0 if isinstance(response_obj, ModelResponse): _usage: Final = getattr(response_obj, "usage", None) if _usage is not None and isinstance(_usage, litellm.Usage): - completion_tokens: Final = _usage.completion_tokens total_tokens = _usage.total_tokens - float(response_ms.total_seconds() / completion_tokens) - # ------------ # Update usage # ------------ diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index a1b67eaeaf9..bb6877a032b 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -1,6 +1,7 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) import random +from collections.abc import Sequence from datetime import datetime, timedelta from typing import TYPE_CHECKING, Any, Final @@ -25,6 +26,12 @@ class RoutingArgs(LiteLLMPydanticObjectBase): max_latency_list_size: int = 10 +def _average_latency(samples: Sequence[float]) -> float: + if not samples: + return 0.0 + return sum(samples) / len(samples) + + class LowestLatencyLoggingHandler(CustomLogger): test_flag: bool = False logged_success: int = 0 @@ -431,23 +438,13 @@ class LowestLatencyLoggingHandler(CustomLogger): item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) # get average latency or average ttft (depending on streaming/non-streaming) - total: float = 0.0 use_ttft = ( request_kwargs is not None and request_kwargs.get("stream", None) is not None and request_kwargs["stream"] is True and len(item_ttft_latency) > 0 ) - if use_ttft: - for _call_latency in item_ttft_latency: - if isinstance(_call_latency, float): - total += _call_latency - item_latency = total / len(item_ttft_latency) - else: - for _call_latency in item_latency: - if isinstance(_call_latency, float): - total += _call_latency - item_latency = total / len(item_latency) + average_latency = _average_latency(item_ttft_latency if use_ttft else item_latency) # -------------- # # Debugging Logic @@ -456,7 +453,7 @@ class LowestLatencyLoggingHandler(CustomLogger): # this helps a user to debug why the router picked a specfic deployment # _deployment_api_base = _deployment.get("litellm_params", {}).get("api_base", "") if _deployment_api_base is not None: - _latency_per_deployment[_deployment_api_base] = item_latency + _latency_per_deployment[_deployment_api_base] = average_latency # -------------- # # End of Debugging Logic # -------------- # @@ -466,7 +463,7 @@ class LowestLatencyLoggingHandler(CustomLogger): ): # if user passed in tpm / rpm in the model_list continue else: - potential_deployments.append((_deployment, item_latency)) + potential_deployments.append((_deployment, average_latency)) if len(potential_deployments) == 0: return None diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py new file mode 100644 index 00000000000..108053dddd9 --- /dev/null +++ b/tests/test_litellm/router_strategy/test_lowest_cost.py @@ -0,0 +1,59 @@ +from datetime import datetime + +import pytest + +import litellm +from litellm.caching.caching import DualCache +from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler + +DEPLOYMENT_ID = "9876" +KWARGS = { + "litellm_params": { + "metadata": {"model_group": "gpt-5.5-pool"}, + "model_info": {"id": DEPLOYMENT_ID}, + } +} + + +def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse: + return litellm.ModelResponse( + model="gpt-5.5", + choices=[{"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}], + usage=litellm.Usage(prompt_tokens=12, completion_tokens=0, total_tokens=12), + ) + + +def _recorded_minute_counters(cache: DualCache) -> dict[str, int]: + cached = cache.get_cache(key="gpt-5.5-pool_map") or {} + minute_buckets = cached.get(DEPLOYMENT_ID, {}) + assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}" + return next(iter(minute_buckets.values())) + + +def test_log_success_event_counts_a_response_with_no_completion_tokens(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + + handler.log_success_event( + kwargs=KWARGS, + response_obj=_chat_response_with_no_completion_tokens(), + start_time=datetime(2026, 1, 1, 12, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 2), + ) + + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} + + +@pytest.mark.asyncio +async def test_async_log_success_event_counts_a_response_with_no_completion_tokens(): + cache = DualCache() + handler = LowestCostLoggingHandler(router_cache=cache) + + await handler.async_log_success_event( + kwargs=KWARGS, + response_obj=_chat_response_with_no_completion_tokens(), + start_time=datetime(2026, 1, 1, 12, 0, 0), + end_time=datetime(2026, 1, 1, 12, 0, 2), + ) + + assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1} diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py index 6701f4a7aa2..eb02459be68 100644 --- a/tests/test_litellm/router_strategy/test_lowest_latency.py +++ b/tests/test_litellm/router_strategy/test_lowest_latency.py @@ -163,3 +163,31 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): assert latencies and latencies[-1] == pytest.approx(2.0) assert not isinstance(latencies[-1], timedelta) json.dumps({"latency": latencies}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "cached_entry", + [{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}], + ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"], +) +async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry): + cache = DualCache() + handler = LowestLatencyLoggingHandler(router_cache=cache) + cache.set_cache( + key="gemini-embedding-001_map", + value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}}, + ) + healthy_deployments = [ + {"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}}, + {"model_info": {"id": "slower"}, "litellm_params": {}}, + ] + + picked = await handler.async_get_available_deployments( + model_group="gemini-embedding-001", + healthy_deployments=healthy_deployments, + request_kwargs={"stream": False, "metadata": {}}, + ) + + assert picked is not None + assert picked["model_info"]["id"] == DEPLOYMENT_ID