mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(health-check): break $0-per-token ties toward per-character-billed rows in the wildcard pick
This commit is contained in:
parent
6fa8a22855
commit
c031573e0e
2 changed files with 46 additions and 14 deletions
|
|
@ -3,6 +3,7 @@ from typing import Final, cast
|
|||
|
||||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.types.utils import ModelInfo
|
||||
|
||||
|
||||
def _form_field_value(value: object) -> str:
|
||||
|
|
@ -155,7 +156,9 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
|||
|
||||
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
|
||||
chat_rows: Final = _chat_row_costs(known_models, custom_llm_provider)
|
||||
return [model for model, _, _ in sorted(chat_rows, key=_billed_rows_cheapest_first_then_free)[:n]]
|
||||
return [
|
||||
model for model, _, _ in sorted(chat_rows, key=_cheapest_per_token_with_per_character_rows_first_on_ties)[:n]
|
||||
]
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
|
||||
|
|
@ -167,16 +170,20 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
|
|||
if model_info.get("mode") != "chat":
|
||||
continue
|
||||
cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
|
||||
yield model, cost, _billed_at_zero(model_info)
|
||||
yield model, cost, _billed_per_character(model_info)
|
||||
|
||||
|
||||
def _billed_at_zero(model_info: Mapping[str, object]) -> bool:
|
||||
return not any(isinstance(value, int | float) and value > 0 for key, value in model_info.items() if "cost" in key)
|
||||
def _billed_per_character(model_info: ModelInfo) -> bool:
|
||||
input_rate: Final = model_info.get("input_cost_per_character") or 0.0
|
||||
output_rate: Final = model_info.get("output_cost_per_character") or 0.0
|
||||
return input_rate > 0 or output_rate > 0
|
||||
|
||||
|
||||
def _billed_rows_cheapest_first_then_free(model_cost: tuple[str, float, bool]) -> tuple[bool, float]:
|
||||
_, cost, free = model_cost
|
||||
return (free, cost)
|
||||
def _cheapest_per_token_with_per_character_rows_first_on_ties(
|
||||
model_cost: tuple[str, float, bool],
|
||||
) -> tuple[float, bool]:
|
||||
_, cost, billed_per_character = model_cost
|
||||
return (cost, not billed_per_character)
|
||||
|
||||
|
||||
def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:
|
||||
|
|
|
|||
|
|
@ -145,24 +145,49 @@ PICKER_PROVIDER: Final = "fake_picker_provider"
|
|||
@pytest.fixture
|
||||
def provider_rows_listed_most_expensive_first(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
rows: Final = {
|
||||
"free-chat": {"input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "mode": "chat"},
|
||||
"pricey-chat": {"input_cost_per_token": 3e-12, "output_cost_per_token": 3e-12, "mode": "chat"},
|
||||
"per-character-and-token-chat": {
|
||||
"input_cost_per_token": 2e-12,
|
||||
"output_cost_per_token": 2e-12,
|
||||
"input_cost_per_character": 1e-12,
|
||||
"mode": "chat",
|
||||
},
|
||||
"cheapest-embedding": {"input_cost_per_token": 1e-13, "mode": "embedding"},
|
||||
"cheap-chat": {"input_cost_per_token": 1e-12, "output_cost_per_token": 1e-12, "mode": "chat"},
|
||||
"per-character-chat": {"input_cost_per_character": 1e-12, "output_cost_per_character": 1e-12, "mode": "chat"},
|
||||
"free-chat": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"input_cost_per_character": 0.0,
|
||||
"output_cost_per_character": 0.0,
|
||||
"mode": "chat",
|
||||
},
|
||||
"per-image-chat": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"output_cost_per_image": 0.08,
|
||||
"mode": "chat",
|
||||
},
|
||||
"input-per-character-chat": {"input_cost_per_character": 1e-12, "mode": "chat"},
|
||||
"output-per-character-chat": {"output_cost_per_character": 1e-12, "mode": "chat"},
|
||||
}
|
||||
for name, row in rows.items():
|
||||
monkeypatch.setitem(litellm.model_cost, name, {**row, "litellm_provider": PICKER_PROVIDER})
|
||||
monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows))
|
||||
|
||||
|
||||
def test_pick_cheapest_chat_models_orders_per_token_cost_with_rows_billed_at_zero_last(
|
||||
def test_pick_cheapest_chat_models_sorts_by_per_token_cost_with_per_character_rows_first_on_ties(
|
||||
provider_rows_listed_most_expensive_first: None,
|
||||
) -> None:
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [
|
||||
"per-character-chat",
|
||||
"cheap-chat",
|
||||
"pricey-chat",
|
||||
"input-per-character-chat",
|
||||
"output-per-character-chat",
|
||||
"free-chat",
|
||||
"per-image-chat",
|
||||
"cheap-chat",
|
||||
"per-character-and-token-chat",
|
||||
"pricey-chat",
|
||||
]
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == [
|
||||
"input-per-character-chat",
|
||||
"output-per-character-chat",
|
||||
]
|
||||
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=2) == ["per-character-chat", "cheap-chat"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue