fix(health): only health-check ids the router sends to the provider

The wildcard health check picker now skips ids that get_llm_provider routes to a different provider or to none, so the longer vertex_ai union cannot pull partner rows into the check. Drops the forced custom_llm_provider and the per-character tie-break from earlier revisions, and pins the replicate and wandb wildcard listing to each row's full owner id.
This commit is contained in:
mateo-berri 2026-10-28 18:48:43 -07:00
parent 47fe64a1e2
commit bf6da246cf
5 changed files with 45 additions and 63 deletions

View file

@ -111,7 +111,7 @@ class HealthCheckHelpers:
model_params["litellm_logging_obj"] = litellm_logging_obj
model_params["fallbacks"] = fallback_models
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
await acompletion(**{**model_params, "custom_llm_provider": custom_llm_provider})
await acompletion(**model_params)
return {}
@staticmethod

View file

@ -1,9 +1,9 @@
from collections.abc import Iterable, Iterator, Mapping
from itertools import islice
from typing import Final, cast
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.types.utils import ModelInfo
def _form_field_value(value: object) -> str:
@ -155,13 +155,11 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
return []
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
chat_rows: Final = _chat_row_costs(known_models, custom_llm_provider)
return [
model for model, _, _ in sorted(chat_rows, key=_cheapest_per_token_with_per_character_rows_first_on_ties)[:n]
]
cheapest_first: Final = sorted(_chat_row_costs(known_models, custom_llm_provider), key=lambda row: row[1])
return list(islice((model for model, _ in cheapest_first if _routes_to(model, custom_llm_provider)), n))
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
for model in models:
try:
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
@ -169,21 +167,15 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
continue
if model_info.get("mode") != "chat":
continue
cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
yield model, cost, _billed_per_character(model_info)
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
def _billed_per_character(model_info: ModelInfo) -> bool:
input_rate: Final = model_info.get("input_cost_per_character") or 0.0
output_rate: Final = model_info.get("output_cost_per_character") or 0.0
return input_rate > 0 or output_rate > 0
def _cheapest_per_token_with_per_character_rows_first_on_ties(
model_cost: tuple[str, float, bool],
) -> tuple[float, bool]:
_, cost, billed_per_character = model_cost
return (cost, not billed_per_character)
def _routes_to(model: str, custom_llm_provider: str) -> bool:
try:
_, routed_provider, _, _ = litellm.get_llm_provider(model=model)
except litellm.BadRequestError:
return False
return routed_provider == custom_llm_provider
def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:

View file

@ -1565,6 +1565,12 @@ CHAT_COMPLETION_BODY: Final = MappingProxyType(
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
)
GEMINI_GENERATE_CONTENT_BODY: Final = MappingProxyType(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2},
}
)
async def test_vertex_wildcard_health_check_only_calls_vertex_hosts(
@ -1575,7 +1581,7 @@ async def test_vertex_wildcard_health_check_only_calls_vertex_hosts(
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
vertex_route: Final = respx_mock.route(host__regex=r".*aiplatform\.googleapis\.com$").respond(
json=dict(CHAT_COMPLETION_BODY)
json=dict(GEMINI_GENERATE_CONTENT_BODY)
)
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))

View file

@ -139,55 +139,28 @@ def test_ensure_extra_body_is_safe_returns_non_dict_unchanged():
assert result is extra_body
PICKER_PROVIDER: Final = "fake_picker_provider"
PICKER_PROVIDER: Final = "deepseek"
@pytest.fixture
def provider_rows_listed_most_expensive_first(monkeypatch: pytest.MonkeyPatch) -> None:
def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyPatch) -> None:
rows: Final = {
"pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"},
"per-character-and-token-chat": {
"input_cost_per_token": 2e-12,
"output_cost_per_token": 2e-12,
"input_cost_per_character": 1e-12,
"mode": "chat",
},
"cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"},
"cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"},
"free-chat": {
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"input_cost_per_character": 0.0,
"output_cost_per_character": 0.0,
"mode": "chat",
},
"per-image-chat": {
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"output_cost_per_image": 0.08,
"mode": "chat",
},
"input-per-character-chat": {"input_cost_per_character": 1e-12, "mode": "chat"},
"output-per-character-chat": {"output_cost_per_character": 1e-12, "mode": "chat"},
"deepseek/fake-pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"},
"deepseek/fake-cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"},
"openai/fake-free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"},
"deepseek/fake-cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"},
"fake-free-chat-without-provider": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"},
}
for name, row in rows.items():
monkeypatch.setitem(litellm.model_cost, name, {**row, "litellm_provider": PICKER_PROVIDER})
monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows))
def test_pick_cheapest_chat_models_sorts_by_per_token_cost_with_per_character_rows_first_on_ties(
provider_rows_listed_most_expensive_first: None,
def test_pick_cheapest_chat_models_offers_only_ids_the_router_sends_to_the_provider(
deepseek_rows_whose_cheapest_ids_route_elsewhere: None,
) -> None:
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [
"input-per-character-chat",
"output-per-character-chat",
"free-chat",
"per-image-chat",
"cheap-chat",
"per-character-and-token-chat",
"pricey-chat",
]
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == [
"input-per-character-chat",
"output-per-character-chat",
"deepseek/fake-cheap-chat",
"deepseek/fake-pricey-chat",
]
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"]

View file

@ -999,7 +999,7 @@ VERTEX_COST_MAP_LABELS: Final = (
@contextmanager
def _registered_vertex_rows(label_per_key: Mapping[str, str]) -> Generator[None, None, None]:
def _registered_rows(label_per_key: Mapping[str, str]) -> Generator[None, None, None]:
import litellm
try:
@ -1022,7 +1022,7 @@ def test_vertex_wildcard_lists_a_row_registered_under_any_vertex_label(label: st
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
key: Final = f"vertex_ai/fake-{label}-row"
with _registered_vertex_rows({key: label}):
with _registered_rows({key: label}):
listed: Final = get_known_models_from_wildcard("vertex_ai/*")
assert key in listed
@ -1036,7 +1036,7 @@ def test_wildcard_keeps_a_publisher_segment_that_names_another_provider(wildcard
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
from litellm.types.router import LiteLLM_Params
with _registered_vertex_rows(
with _registered_rows(
{
"vertex_ai/openai/fake-publisher-maas": "vertex_ai-openai_models",
"vertex_ai/meta/fake-publisher-maas": "vertex_ai-llama_models",
@ -1052,6 +1052,17 @@ def test_wildcard_keeps_a_publisher_segment_that_names_another_provider(wildcard
assert f"{listed_prefix}/fake-publisher-maas" not in listed
@pytest.mark.parametrize("provider", ["replicate", "wandb"])
def test_wildcard_lists_each_model_once_under_its_full_owner_id(provider: str):
import litellm
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
full_ids: Final = {f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]}
assert any(model.count("/") > 1 for model in full_ids)
assert sorted(get_known_models_from_wildcard(f"{provider}/*")) == sorted(full_ids)
def test_get_complete_model_list_drops_no_default_models_sentinel():
from litellm.proxy.auth.model_checks import get_complete_model_list