From bf6da246cfed707126f59186e122020757006e5f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:48:43 -0700 Subject: [PATCH] fix(health): only health-check ids the router sends to the provider The wildcard health check picker now skips ids that get_llm_provider routes to a different provider or to none, so the longer vertex_ai union cannot pull partner rows into the check. Drops the forced custom_llm_provider and the per-character tie-break from earlier revisions, and pins the replicate and wandb wildcard listing to each row's full owner id. --- .../health_check_helpers.py | 2 +- .../litellm_core_utils/llm_request_utils.py | 30 ++++------- .../test_health_check_helpers.py | 8 ++- .../test_llm_request_utils.py | 51 +++++-------------- tests/unit/proxy/auth/test_model_checks.py | 17 +++++-- 5 files changed, 45 insertions(+), 63 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 0574a8c2f16..b19bcd941fa 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -111,7 +111,7 @@ class HealthCheckHelpers: model_params["litellm_logging_obj"] = litellm_logging_obj model_params["fallbacks"] = fallback_models model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16 - await acompletion(**{**model_params, "custom_llm_provider": custom_llm_provider}) + await acompletion(**model_params) return {} @staticmethod diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index d0c0eac99af..a03467d9d3c 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,9 +1,9 @@ from collections.abc import Iterable, Iterator, Mapping +from itertools import islice from typing import Final, cast import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH -from litellm.types.utils import ModelInfo def _form_field_value(value: object) -> str: @@ -155,13 +155,11 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): return [] known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) - chat_rows: Final = _chat_row_costs(known_models, custom_llm_provider) - return [ - model for model, _, _ in sorted(chat_rows, key=_cheapest_per_token_with_per_character_rows_first_on_ties)[:n] - ] + cheapest_first: Final = sorted(_chat_row_costs(known_models, custom_llm_provider), key=lambda row: row[1]) + return list(islice((model for model, _ in cheapest_first if _routes_to(model, custom_llm_provider)), n)) -def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]: +def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: for model in models: try: model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider) @@ -169,21 +167,15 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator continue if model_info.get("mode") != "chat": continue - cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - yield model, cost, _billed_per_character(model_info) + yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) -def _billed_per_character(model_info: ModelInfo) -> bool: - input_rate: Final = model_info.get("input_cost_per_character") or 0.0 - output_rate: Final = model_info.get("output_cost_per_character") or 0.0 - return input_rate > 0 or output_rate > 0 - - -def _cheapest_per_token_with_per_character_rows_first_on_ties( - model_cost: tuple[str, float, bool], -) -> tuple[float, bool]: - _, cost, billed_per_character = model_cost - return (cost, not billed_per_character) +def _routes_to(model: str, custom_llm_provider: str) -> bool: + try: + _, routed_provider, _, _ = litellm.get_llm_provider(model=model) + except litellm.BadRequestError: + return False + return routed_provider == custom_llm_provider def get_proxy_server_request_headers(litellm_params: dict | None) -> dict: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 4a17ef3a42d..5ca8d692690 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1565,6 +1565,12 @@ CHAT_COMPLETION_BODY: Final = MappingProxyType( "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } ) +GEMINI_GENERATE_CONTENT_BODY: Final = MappingProxyType( + { + "candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + } +) async def test_vertex_wildcard_health_check_only_calls_vertex_hosts( @@ -1575,7 +1581,7 @@ async def test_vertex_wildcard_health_check_only_calls_vertex_hosts( monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() vertex_route: Final = respx_mock.route(host__regex=r".*aiplatform\.googleapis\.com$").respond( - json=dict(CHAT_COMPLETION_BODY) + json=dict(GEMINI_GENERATE_CONTENT_BODY) ) other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY)) diff --git a/tests/unit/litellm_core_utils/test_llm_request_utils.py b/tests/unit/litellm_core_utils/test_llm_request_utils.py index a1fd8ebe78e..20bbfa7ce82 100644 --- a/tests/unit/litellm_core_utils/test_llm_request_utils.py +++ b/tests/unit/litellm_core_utils/test_llm_request_utils.py @@ -139,55 +139,28 @@ def test_ensure_extra_body_is_safe_returns_non_dict_unchanged(): assert result is extra_body -PICKER_PROVIDER: Final = "fake_picker_provider" +PICKER_PROVIDER: Final = "deepseek" @pytest.fixture -def provider_rows_listed_most_expensive_first(monkeypatch: pytest.MonkeyPatch) -> None: +def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyPatch) -> None: rows: Final = { - "pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"}, - "per-character-and-token-chat": { - "input_cost_per_token": 2e-12, - "output_cost_per_token": 2e-12, - "input_cost_per_character": 1e-12, - "mode": "chat", - }, - "cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"}, - "cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"}, - "free-chat": { - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "input_cost_per_character": 0.0, - "output_cost_per_character": 0.0, - "mode": "chat", - }, - "per-image-chat": { - "input_cost_per_token": 0.0, - "output_cost_per_token": 0.0, - "output_cost_per_image": 0.08, - "mode": "chat", - }, - "input-per-character-chat": {"input_cost_per_character": 1e-12, "mode": "chat"}, - "output-per-character-chat": {"output_cost_per_character": 1e-12, "mode": "chat"}, + "deepseek/fake-pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"}, + "deepseek/fake-cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"}, + "openai/fake-free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, + "deepseek/fake-cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"}, + "fake-free-chat-without-provider": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, } for name, row in rows.items(): monkeypatch.setitem(litellm.model_cost, name, {**row, "litellm_provider": PICKER_PROVIDER}) monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows)) -def test_pick_cheapest_chat_models_sorts_by_per_token_cost_with_per_character_rows_first_on_ties( - provider_rows_listed_most_expensive_first: None, +def test_pick_cheapest_chat_models_offers_only_ids_the_router_sends_to_the_provider( + deepseek_rows_whose_cheapest_ids_route_elsewhere: None, ) -> None: assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [ - "input-per-character-chat", - "output-per-character-chat", - "free-chat", - "per-image-chat", - "cheap-chat", - "per-character-and-token-chat", - "pricey-chat", - ] - assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == [ - "input-per-character-chat", - "output-per-character-chat", + "deepseek/fake-cheap-chat", + "deepseek/fake-pricey-chat", ] + assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"] diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index f14f7688546..3f0f97e63b0 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -999,7 +999,7 @@ VERTEX_COST_MAP_LABELS: Final = ( @contextmanager -def _registered_vertex_rows(label_per_key: Mapping[str, str]) -> Generator[None, None, None]: +def _registered_rows(label_per_key: Mapping[str, str]) -> Generator[None, None, None]: import litellm try: @@ -1022,7 +1022,7 @@ def test_vertex_wildcard_lists_a_row_registered_under_any_vertex_label(label: st from litellm.proxy.auth.model_checks import get_known_models_from_wildcard key: Final = f"vertex_ai/fake-{label}-row" - with _registered_vertex_rows({key: label}): + with _registered_rows({key: label}): listed: Final = get_known_models_from_wildcard("vertex_ai/*") assert key in listed @@ -1036,7 +1036,7 @@ def test_wildcard_keeps_a_publisher_segment_that_names_another_provider(wildcard from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.types.router import LiteLLM_Params - with _registered_vertex_rows( + with _registered_rows( { "vertex_ai/openai/fake-publisher-maas": "vertex_ai-openai_models", "vertex_ai/meta/fake-publisher-maas": "vertex_ai-llama_models", @@ -1052,6 +1052,17 @@ def test_wildcard_keeps_a_publisher_segment_that_names_another_provider(wildcard assert f"{listed_prefix}/fake-publisher-maas" not in listed +@pytest.mark.parametrize("provider", ["replicate", "wandb"]) +def test_wildcard_lists_each_model_once_under_its_full_owner_id(provider: str): + import litellm + from litellm.proxy.auth.model_checks import get_known_models_from_wildcard + + full_ids: Final = {f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]} + assert any(model.count("/") > 1 for model in full_ids) + + assert sorted(get_known_models_from_wildcard(f"{provider}/*")) == sorted(full_ids) + + def test_get_complete_model_list_drops_no_default_models_sentinel(): from litellm.proxy.auth.model_checks import get_complete_model_list