From c031573e0eefabecfe898c72c7cab201ac932f5c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:49:56 -0700 Subject: [PATCH] fix(health-check): break $0-per-token ties toward per-character-billed rows in the wildcard pick --- .../litellm_core_utils/llm_request_utils.py | 21 ++++++---- .../test_llm_request_utils.py | 39 +++++++++++++++---- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 4a777992660..d0c0eac99af 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -3,6 +3,7 @@ from typing import Final, cast import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.types.utils import ModelInfo def _form_field_value(value: object) -> str: @@ -155,7 +156,9 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) chat_rows: Final = _chat_row_costs(known_models, custom_llm_provider) - return [model for model, _, _ in sorted(chat_rows, key=_billed_rows_cheapest_first_then_free)[:n]] + return [ + model for model, _, _ in sorted(chat_rows, key=_cheapest_per_token_with_per_character_rows_first_on_ties)[:n] + ] def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]: @@ -167,16 +170,20 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator if model_info.get("mode") != "chat": continue cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) - yield model, cost, _billed_at_zero(model_info) + yield model, cost, _billed_per_character(model_info) -def _billed_at_zero(model_info: Mapping[str, object]) -> bool: - return not any(isinstance(value, int | float) and value > 0 for key, value in model_info.items() if "cost" in key) +def _billed_per_character(model_info: ModelInfo) -> bool: + input_rate: Final = model_info.get("input_cost_per_character") or 0.0 + output_rate: Final = model_info.get("output_cost_per_character") or 0.0 + return input_rate > 0 or output_rate > 0 -def _billed_rows_cheapest_first_then_free(model_cost: tuple[str, float, bool]) -> tuple[bool, float]: - _, cost, free = model_cost - return (free, cost) +def _cheapest_per_token_with_per_character_rows_first_on_ties( + model_cost: tuple[str, float, bool], +) -> tuple[float, bool]: + _, cost, billed_per_character = model_cost + return (cost, not billed_per_character) def get_proxy_server_request_headers(litellm_params: dict | None) -> dict: diff --git a/tests/unit/litellm_core_utils/test_llm_request_utils.py b/tests/unit/litellm_core_utils/test_llm_request_utils.py index 18d650eb00c..a1fd8ebe78e 100644 --- a/tests/unit/litellm_core_utils/test_llm_request_utils.py +++ b/tests/unit/litellm_core_utils/test_llm_request_utils.py @@ -145,24 +145,49 @@ PICKER_PROVIDER: Final = "fake_picker_provider" @pytest.fixture def provider_rows_listed_most_expensive_first(monkeypatch: pytest.MonkeyPatch) -> None: rows: Final = { - "free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"}, "pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"}, + "per-character-and-token-chat": { + "input_cost_per_token": 2e-12, + "output_cost_per_token": 2e-12, + "input_cost_per_character": 1e-12, + "mode": "chat", + }, "cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"}, "cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"}, - "per-character-chat": {"input_cost_per_character": 1e-12, "output_cost_per_character": 1e-12, "mode": "chat"}, + "free-chat": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "input_cost_per_character": 0.0, + "output_cost_per_character": 0.0, + "mode": "chat", + }, + "per-image-chat": { + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "output_cost_per_image": 0.08, + "mode": "chat", + }, + "input-per-character-chat": {"input_cost_per_character": 1e-12, "mode": "chat"}, + "output-per-character-chat": {"output_cost_per_character": 1e-12, "mode": "chat"}, } for name, row in rows.items(): monkeypatch.setitem(litellm.model_cost, name, {**row, "litellm_provider": PICKER_PROVIDER}) monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows)) -def test_pick_cheapest_chat_models_orders_per_token_cost_with_rows_billed_at_zero_last( +def test_pick_cheapest_chat_models_sorts_by_per_token_cost_with_per_character_rows_first_on_ties( provider_rows_listed_most_expensive_first: None, ) -> None: assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [ - "per-character-chat", - "cheap-chat", - "pricey-chat", + "input-per-character-chat", + "output-per-character-chat", "free-chat", + "per-image-chat", + "cheap-chat", + "per-character-and-token-chat", + "pricey-chat", + ] + assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == [ + "input-per-character-chat", + "output-per-character-chat", ] - assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == ["per-character-chat", "cheap-chat"]