mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): only health-check ids the router sends to the provider
The wildcard health check picker now skips ids that get_llm_provider routes to a different provider or to none, so the longer vertex_ai union cannot pull partner rows into the check. Drops the forced custom_llm_provider and the per-character tie-break from earlier revisions, and pins the replicate and wandb wildcard listing to each row's full owner id.
This commit is contained in:
parent
47fe64a1e2
commit
bf6da246cf
5 changed files with 45 additions and 63 deletions
|
|
@ -111,7 +111,7 @@ class HealthCheckHelpers:
|
|||
model_params["litellm_logging_obj"] = litellm_logging_obj
|
||||
model_params["fallbacks"] = fallback_models
|
||||
model_params["max_tokens"] = model_params.get("max_tokens", 16) # GPT-5 models require max_output_tokens >= 16
|
||||
await acompletion(**{**model_params, "custom_llm_provider": custom_llm_provider})
|
||||
await acompletion(**model_params)
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from itertools import islice
|
||||
from typing import Final, cast
|
||||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
def _form_field_value(value: object) -> str:
|
||||
|
|
@ -155,13 +155,11 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
|||
return []
|
||||
|
||||
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
|
||||
chat_rows: Final = _chat_row_costs(known_models, custom_llm_provider)
|
||||
return [
|
||||
model for model, _, _ in sorted(chat_rows, key=_cheapest_per_token_with_per_character_rows_first_on_ties)[:n]
|
||||
]
|
||||
cheapest_first: Final = sorted(_chat_row_costs(known_models, custom_llm_provider), key=lambda row: row[1])
|
||||
return list(islice((model for model, _ in cheapest_first if _routes_to(model, custom_llm_provider)), n))
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
|
||||
for model in models:
|
||||
try:
|
||||
model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
|
@ -169,21 +167,15 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
|
|||
continue
|
||||
if model_info.get("mode") != "chat":
|
||||
continue
|
||||
cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
|
||||
yield model, cost, _billed_per_character(model_info)
|
||||
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
|
||||
|
||||
|
||||
def _billed_per_character(model_info: ModelInfo) -> bool:
|
||||
input_rate: Final = model_info.get("input_cost_per_character") or 0.0
|
||||
output_rate: Final = model_info.get("output_cost_per_character") or 0.0
|
||||
return input_rate > 0 or output_rate > 0
|
||||
|
||||
|
||||
def _cheapest_per_token_with_per_character_rows_first_on_ties(
|
||||
model_cost: tuple[str, float, bool],
|
||||
) -> tuple[float, bool]:
|
||||
_, cost, billed_per_character = model_cost
|
||||
return (cost, not billed_per_character)
|
||||
def _routes_to(model: str, custom_llm_provider: str) -> bool:
|
||||
try:
|
||||
_, routed_provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except litellm.BadRequestError:
|
||||
return False
|
||||
return routed_provider == custom_llm_provider
|
||||
|
||||
|
||||
def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:
|
||||
|
|
|
|||
|
|
@ -1565,6 +1565,12 @@ CHAT_COMPLETION_BODY: Final = MappingProxyType(
|
|||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
)
|
||||
GEMINI_GENERATE_CONTENT_BODY: Final = MappingProxyType(
|
||||
{
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "ok"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def test_vertex_wildcard_health_check_only_calls_vertex_hosts(
|
||||
|
|
@ -1575,7 +1581,7 @@ async def test_vertex_wildcard_health_check_only_calls_vertex_hosts(
|
|||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
vertex_route: Final = respx_mock.route(host__regex=r".*aiplatform\.googleapis\.com$").respond(
|
||||
json=dict(CHAT_COMPLETION_BODY)
|
||||
json=dict(GEMINI_GENERATE_CONTENT_BODY)
|
||||
)
|
||||
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))
|
||||
|
||||
|
|
|
|||
|
|
@ -139,55 +139,28 @@ def test_ensure_extra_body_is_safe_returns_non_dict_unchanged():
|
|||
assert result is extra_body
|
||||
|
||||
|
||||
PICKER_PROVIDER: Final = "fake_picker_provider"
|
||||
PICKER_PROVIDER: Final = "deepseek"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def provider_rows_listed_most_expensive_first(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
rows: Final = {
|
||||
"pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"},
|
||||
"per-character-and-token-chat": {
|
||||
"input_cost_per_token": 2e-12,
|
||||
"output_cost_per_token": 2e-12,
|
||||
"input_cost_per_character": 1e-12,
|
||||
"mode": "chat",
|
||||
},
|
||||
"cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"},
|
||||
"cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"},
|
||||
"free-chat": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"input_cost_per_character": 0.0,
|
||||
"output_cost_per_character": 0.0,
|
||||
"mode": "chat",
|
||||
},
|
||||
"per-image-chat": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_cost_per_image": 0.08,
|
||||
"mode": "chat",
|
||||
},
|
||||
"input-per-character-chat": {"input_cost_per_character": 1e-12, "mode": "chat"},
|
||||
"output-per-character-chat": {"output_cost_per_character": 1e-12, "mode": "chat"},
|
||||
"deepseek/fake-pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"},
|
||||
"deepseek/fake-cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"},
|
||||
"openai/fake-free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"},
|
||||
"deepseek/fake-cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"},
|
||||
"fake-free-chat-without-provider": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"},
|
||||
}
|
||||
for name, row in rows.items():
|
||||
monkeypatch.setitem(litellm.model_cost, name, {**row, "litellm_provider": PICKER_PROVIDER})
|
||||
monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows))
|
||||
|
||||
|
||||
def test_pick_cheapest_chat_models_sorts_by_per_token_cost_with_per_character_rows_first_on_ties(
|
||||
provider_rows_listed_most_expensive_first: None,
|
||||
def test_pick_cheapest_chat_models_offers_only_ids_the_router_sends_to_the_provider(
|
||||
deepseek_rows_whose_cheapest_ids_route_elsewhere: None,
|
||||
) -> None:
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [
|
||||
"input-per-character-chat",
|
||||
"output-per-character-chat",
|
||||
"free-chat",
|
||||
"per-image-chat",
|
||||
"cheap-chat",
|
||||
"per-character-and-token-chat",
|
||||
"pricey-chat",
|
||||
]
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == [
|
||||
"input-per-character-chat",
|
||||
"output-per-character-chat",
|
||||
"deepseek/fake-cheap-chat",
|
||||
"deepseek/fake-pricey-chat",
|
||||
]
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"]
|
||||
|
|
|
|||
|
|
@ -999,7 +999,7 @@ VERTEX_COST_MAP_LABELS: Final = (
|
|||
|
||||
|
||||
@contextmanager
|
||||
def _registered_vertex_rows(label_per_key: Mapping[str, str]) -> Generator[None, None, None]:
|
||||
def _registered_rows(label_per_key: Mapping[str, str]) -> Generator[None, None, None]:
|
||||
import litellm
|
||||
|
||||
try:
|
||||
|
|
@ -1022,7 +1022,7 @@ def test_vertex_wildcard_lists_a_row_registered_under_any_vertex_label(label: st
|
|||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
|
||||
key: Final = f"vertex_ai/fake-{label}-row"
|
||||
with _registered_vertex_rows({key: label}):
|
||||
with _registered_rows({key: label}):
|
||||
listed: Final = get_known_models_from_wildcard("vertex_ai/*")
|
||||
|
||||
assert key in listed
|
||||
|
|
@ -1036,7 +1036,7 @@ def test_wildcard_keeps_a_publisher_segment_that_names_another_provider(wildcard
|
|||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.types.router import LiteLLM_Params
|
||||
|
||||
with _registered_vertex_rows(
|
||||
with _registered_rows(
|
||||
{
|
||||
"vertex_ai/openai/fake-publisher-maas": "vertex_ai-openai_models",
|
||||
"vertex_ai/meta/fake-publisher-maas": "vertex_ai-llama_models",
|
||||
|
|
@ -1052,6 +1052,17 @@ def test_wildcard_keeps_a_publisher_segment_that_names_another_provider(wildcard
|
|||
assert f"{listed_prefix}/fake-publisher-maas" not in listed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["replicate", "wandb"])
|
||||
def test_wildcard_lists_each_model_once_under_its_full_owner_id(provider: str):
|
||||
import litellm
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
|
||||
full_ids: Final = {f"{provider}/{model.removeprefix(f'{provider}/')}" for model in litellm.models_by_provider[provider]}
|
||||
assert any(model.count("/") > 1 for model in full_ids)
|
||||
|
||||
assert sorted(get_known_models_from_wildcard(f"{provider}/*")) == sorted(full_ids)
|
||||
|
||||
|
||||
def test_get_complete_model_list_drops_no_default_models_sentinel():
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue