mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(health): keep wildcard health picks that only route through a saved custom_llm_provider
Rank picks routed to the checked provider first and ids routed to no provider second, each by cost, and skip only ids routed to a different provider, so a wildcard saved with custom_llm_provider still sends its health request
This commit is contained in:
parent
e598104c7a
commit
20880a6731
3 changed files with 39 additions and 6 deletions
|
|
@ -1,5 +1,4 @@
|
|||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from itertools import islice
|
||||
from typing import Final, cast
|
||||
|
||||
import litellm
|
||||
|
|
@ -156,8 +155,12 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
|||
return []
|
||||
|
||||
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
|
||||
cheapest_first: Final = sorted(_chat_row_costs(known_models, custom_llm_provider), key=lambda row: row[1])
|
||||
return list(islice((model for model, _ in cheapest_first if _routes_to(model, custom_llm_provider)), n))
|
||||
ranked_rows: Final = (
|
||||
(route_rank, cost, model)
|
||||
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
|
||||
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
|
||||
)
|
||||
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
|
||||
|
|
@ -171,9 +174,13 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
|
|||
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
|
||||
|
||||
|
||||
def _routes_to(model: str, custom_llm_provider: str) -> bool:
|
||||
def _route_rank(model: str, custom_llm_provider: str) -> int | None:
|
||||
routed_provider: Final = inferred_provider(model)
|
||||
return routed_provider is not None and routed_provider == inferred_provider(f"{custom_llm_provider}/{model}")
|
||||
if routed_provider is None:
|
||||
return 1
|
||||
if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"):
|
||||
return 0
|
||||
return None
|
||||
|
||||
|
||||
def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:
|
||||
|
|
|
|||
|
|
@ -1620,3 +1620,28 @@ async def test_ai21_wildcard_health_check_calls_ai21(
|
|||
assert ai21_route.called
|
||||
assert not other_hosts.called
|
||||
|
||||
|
||||
async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_a_routing_prefix(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"fake-together-chat",
|
||||
{"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9, "mode": "chat", "litellm_provider": "together_ai"},
|
||||
)
|
||||
monkeypatch.setitem(litellm.models_by_provider, "together_ai", ("fake-together-chat",))
|
||||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
together_route: Final = respx_mock.route(host="api.together.ai").respond(json=dict(CHAT_COMPLETION_BODY))
|
||||
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))
|
||||
|
||||
result: Final = await ahealth_check(
|
||||
model_params={"model": "together_ai/*", "custom_llm_provider": "together_ai", "api_key": "health-together-key"},
|
||||
mode="chat",
|
||||
)
|
||||
|
||||
assert "error" not in result, result
|
||||
assert json.loads(together_route.calls.last.request.content)["model"] == "fake-together-chat"
|
||||
assert not other_hosts.called
|
||||
|
||||
|
|
|
|||
|
|
@ -156,11 +156,12 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP
|
|||
monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows))
|
||||
|
||||
|
||||
def test_pick_cheapest_chat_models_offers_only_ids_the_router_sends_to_the_provider(
|
||||
def test_pick_cheapest_chat_models_prefers_provider_routed_ids_and_skips_ids_routed_elsewhere(
|
||||
deepseek_rows_whose_cheapest_ids_route_elsewhere: None,
|
||||
) -> None:
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [
|
||||
"deepseek/fake-cheap-chat",
|
||||
"deepseek/fake-pricey-chat",
|
||||
"fake-free-chat-without-provider",
|
||||
]
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue