diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b5dc0ce13de..4a2df688735 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -909,7 +909,7 @@ def _bedrock_pricing_info( def cost_map_key_prices_anything(key: str) -> bool: - entry: Final = _raw_cost_map_entry(key) + entry: Final = raw_cost_map_entry(key) return entry is not None and _cost_map_entry_prices_anything(entry) @@ -2351,7 +2351,7 @@ def _deployment_model_info( ) -> ModelInfo | None: if not custom_pricing: return None - registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None + registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None if registered_deployment_info is not None: return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id if litellm_logging_obj is None: @@ -2424,7 +2424,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, . ) -def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None: +def raw_cost_map_entry(key: str) -> Mapping[str, object] | None: raw_entry: Final = litellm.model_cost.get(key) return raw_entry if isinstance(raw_entry, Mapping) else None @@ -2468,7 +2468,7 @@ def pricing_entry_for_cost_calc( ) if resolved is None: return None - return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved + return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved def ocr_cost( diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b0b875af6d..5f5d5b4b920 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -910,7 +910,9 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: return 0.0 -def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None: +def get_cost_per_unit( + model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0 +) -> float | None: # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml) cost_per_unit: Final = model_info.get(cost_key) if isinstance(cost_per_unit, float): @@ -954,6 +956,33 @@ def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float _get_cost_per_unit = get_cost_per_unit +_HALF_PRICED_BATCH_KEYS: Final = frozenset( + {f"input_cost_per_token{_BATCH_KEY_SUFFIX}", f"output_cost_per_token{_BATCH_KEY_SUFFIX}"} +) +_TIER_KEY_SUFFIXES: Final = ("", *SERVICE_TIER_COST_KEY_SUFFIXES) +_CACHE_KEYS_BILLED_AT_INPUT_RATE: Final = MappingProxyType( + { + **{f"cache_read_input_token_cost{suffix}": f"input_cost_per_token{suffix}" for suffix in _TIER_KEY_SUFFIXES}, + **{ + f"cache_creation_input_token_cost{suffix}": f"input_cost_per_token{suffix}" for suffix in _TIER_KEY_SUFFIXES + }, + } +) + + +def billed_rate(model_info: Mapping[str, object], cost_key: str) -> float | None: + rate: Final = get_cost_per_unit(model_info, cost_key, default_value=None) + if rate is not None: + return rate + if cost_key in _HALF_PRICED_BATCH_KEYS: + standard_rate: Final = get_cost_per_unit( + model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None + ) + return None if standard_rate is None else standard_rate / 2 + input_key: Final = _CACHE_KEYS_BILLED_AT_INPUT_RATE.get(cost_key) + return None if input_key is None else get_cost_per_unit(model_info, input_key, default_value=None) + + def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None: """The prices a deployment sets itself, as floats; None when it sets none that parse.""" if model_info is None: diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index aa6ddae438d..6d7098d6b5a 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -2,7 +2,7 @@ import importlib.util import os import re import sys -from collections.abc import Collection, Iterator, Mapping +from collections.abc import Collection, Iterator, Mapping, Sequence from functools import lru_cache from itertools import accumulate, chain, repeat from logging import Logger @@ -38,7 +38,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_ENDPOINT_MARKER, ) from litellm.types.router import ( - CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, + ConfigurableClientsideParamsCustomAuth, Deployment, reject_server_owned_wif_fields, server_owned_wif_fields_present, @@ -189,7 +189,7 @@ def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool: def _is_param_allowed( param: str, request_body_value: object, - configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, + configurable_clientside_auth_params: Sequence[str | ConfigurableClientsideParamsCustomAuth] | None, ) -> bool: """ Check if param is a str or dict and if request_body_value is in the list of allowed values diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 08840b44532..572bddec1e8 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -45,7 +45,7 @@ from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncreme from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget -from litellm.types.router import DeploymentTypedDict +from litellm.types.router import DeploymentTypedDict, ModelGroupInfo @dataclass @@ -706,14 +706,7 @@ def _model_access_group_counter(group: str, budget: ModelAccessGroupBudget | Non def _dedupe_tags(tags: list[str]) -> list[str]: - seen: Final = set() - deduped_tags: Final = [] - for tag in tags: - if tag in seen: - continue - seen.add(tag) - deduped_tags.append(tag) - return deduped_tags + return list(dict.fromkeys(tags)) @with_service_target(AUTH_OBJECTS_TARGET) @@ -1507,6 +1500,9 @@ def _estimate_image_generation_cost( return cost_per_image * max(n, 1) +_DECLARED_GROUP_FIELDS: Final = MappingProxyType(dict.fromkeys(ModelGroupInfo.model_fields, True)) + + def _get_model_cost_info( model: str, llm_router: Router | None, @@ -1514,7 +1510,7 @@ def _get_model_cost_info( if llm_router is not None: model_group_info: Final = llm_router.cached_model_group_info(model) if model_group_info is not None: - return model_group_info.model_dump() + return model_group_info.model_dump(include=_DECLARED_GROUP_FIELDS) return dict(litellm.get_model_info(model=model)) diff --git a/litellm/router.py b/litellm/router.py index 3809a5d6555..963e28cf1fb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -103,6 +103,7 @@ from litellm.constants import ( RUNTIME_UPDATABLE_ROUTER_SETTINGS, SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, ) +from litellm.cost_calculator import raw_cost_map_entry from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes @@ -142,7 +143,8 @@ from litellm.litellm_core_utils.hidden_params import ( set_hidden_params, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging -from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES +from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model +from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES, billed_rate from litellm.litellm_core_utils.ptu_pricing import ( PTU_COST_ATTRIBUTION_ENV_VAR, declares_ptu, @@ -454,6 +456,9 @@ from litellm.types.router import ( SearchToolTypedDict, TaggedPreRoutingStrategy, holds_secret_pointer, + is_usd_price_key, + model_info_prices, + price_as_float, ) from litellm.types.services import ServiceTypes from litellm.types.utils import ( @@ -547,15 +552,6 @@ RouterStrategySelector: TypeAlias = ( ) -def _cost_value_as_float(value: str | float | None) -> float | None: - if value is None: - return None - try: - return float(value) - except (TypeError, ValueError): - return None - - def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool: """Single owner of the environment-gating rule: a deployment whose model_info names `supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list. @@ -724,6 +720,57 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {}) +def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, object]: + configured_info: Final = _configured_model_info(deployment) + model_id: Final = configured_info.get("id") + custom_priced: Final = use_custom_pricing_for_model( + { + **_LITELLM_PARAMS_ADAPTER.validate_python(deployment.get("litellm_params") or {}), + "metadata": {"model_info": dict(configured_info)}, + } + ) + registered_entry: Final = raw_cost_map_entry(model_id) if custom_priced and isinstance(model_id, str) else None + if registered_entry is not None: + return registered_entry + cost_map_key: Final = model_info.get("key") + catalog_entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None + return model_info if catalog_entry is None else {**catalog_entry, **model_info} + + +def _deployment_price_entry( + deployment: Mapping[str, object], model_info: Mapping[str, object] | None +) -> Mapping[str, object]: + if model_info is None: + return _configured_model_info(deployment) + token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")} + return {**_billed_pricing_entry(deployment, model_info), **token_prices} + + +@dataclass(frozen=True, slots=True) +class _GroupDeployment: + model_info: ModelMapInfo | None + price_entry: Mapping[str, object] + team_may_use: bool + + def listed_prices(self) -> Mapping[str, float]: + prices: Final = model_info_prices(self.price_entry) + if self.team_may_use: + return prices + return MappingProxyType({key: price for key, price in prices.items() if key in ModelGroupInfo.model_fields}) + + +def _highest_billed_rate(cost_key: str, price_entries: Sequence[Mapping[str, object]]) -> float | None: + rates: Final = (price_as_float(billed_rate(entry, cost_key)) for entry in price_entries) + return max((rate for rate in rates if rate is not None), default=None) + + +def _highest_billed_prices( + listed_keys: Iterable[str], price_entries: Sequence[Mapping[str, object]] +) -> Mapping[str, float]: + rates: Final = ((key, _highest_billed_rate(key, price_entries)) for key in listed_keys) + return MappingProxyType({key: rate for key, rate in rates if rate is not None}) + + def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None: from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request @@ -12676,17 +12723,12 @@ class Router: (d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)), None, ) - for model in model_list: - is_match = False - if ( - "model_name" in model - and model["model_name"] == model_group - or "model_name" in model - and self.pattern_router.route(model_group) is not None - ): # exact match - is_match = True - - if not is_match: + group_deployments: Final = tuple( + self._group_deployment(model, team_id) if self._is_model_group_member(model, model_group) else None + for model in model_list + ) + for model, group_deployment in zip(model_list, group_deployments, strict=True): + if group_deployment is None: continue # model in model group found # litellm_params = LiteLLM_Params(**model["litellm_params"]) @@ -12727,17 +12769,9 @@ class Router: if _deployment_otpm is None: _deployment_otpm = model_info_dict.get("otpm", None) - # get model info - try: - model_id = model_info_dict.get("id", None) - if model_id is not None: - model_info = self.get_deployment_model_info(model_id=model_id, model_name=litellm_params.model) - else: - model_info = None - except Exception: - model_info = None - + model_info = group_deployment.model_info deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict) + deployment_prices = group_deployment.listed_prices() # get llm provider litellm_model, llm_provider = self._model_group_llm_provider(litellm_params) @@ -12752,8 +12786,8 @@ class Router: # Get mode from database model_info if available, otherwise default to "chat" db_model_info = model.get("model_info", {}) mode = db_model_info.get("mode", "chat") - input_cost_per_token = _cost_value_as_float(db_model_info.get("input_cost_per_token")) - output_cost_per_token = _cost_value_as_float(db_model_info.get("output_cost_per_token")) + input_cost_per_token = price_as_float(db_model_info.get("input_cost_per_token")) + output_cost_per_token = price_as_float(db_model_info.get("output_cost_per_token")) model_info = ModelMapInfo( key=model_group, @@ -12774,7 +12808,8 @@ class Router: "model_group": user_facing_model_group_name, "providers": [llm_provider], "description": model_group_description, - **model_info, + **{key: value for key, value in model_info.items() if not is_usd_price_key(key)}, + **deployment_prices, "supports_fast_mode": True, "supported_reasoning_efforts": None, "supported_endpoints": None, @@ -12808,18 +12843,7 @@ class Router: ) ): model_group_info.max_output_tokens = model_info["max_output_tokens"] - _input_cost_per_token = _cost_value_as_float(model_info.get("input_cost_per_token")) - if _input_cost_per_token is not None and ( - model_group_info.input_cost_per_token is None - or _input_cost_per_token > (model_group_info.input_cost_per_token or 0.0) - ): - model_group_info.input_cost_per_token = _input_cost_per_token - _output_cost_per_token = _cost_value_as_float(model_info.get("output_cost_per_token")) - if _output_cost_per_token is not None and ( - model_group_info.output_cost_per_token is None - or _output_cost_per_token > (model_group_info.output_cost_per_token or 0.0) - ): - model_group_info.output_cost_per_token = _output_cost_per_token + model_group_info = model_group_info.with_higher_prices(deployment_prices) if ( model_info.get("supports_parallel_function_calling", None) is not None and model_info["supports_parallel_function_calling"] is True @@ -12864,7 +12888,7 @@ class Router: deployment_supported_endpoints( model_info.get("supported_endpoints"), model, llm_provider ) - if team_may_use_deployment(_configured_model_info(model).get("team_id"), team_id) + if group_deployment.team_may_use else () ), ) @@ -12928,7 +12952,39 @@ class Router: if configurable_clientside_auth_params is not None: model_group_info.configurable_clientside_auth_params = configurable_clientside_auth_params - return model_group_info + usable_price_entries: Final = tuple( + deployment.price_entry + for deployment in group_deployments + if deployment is not None and deployment.team_may_use + ) + return model_group_info.with_higher_prices( + _highest_billed_prices(model_info_prices(model_group_info.model_dump()), usable_price_entries) + ) + + return None + + def _is_model_group_member(self, deployment: DeploymentTypedDict, model_group: str) -> bool: + return "model_name" in deployment and ( + deployment["model_name"] == model_group or self.pattern_router.route(model_group) is not None + ) + + def _deployment_model_info(self, deployment: DeploymentTypedDict) -> ModelMapInfo | None: + model_id: Final = _configured_model_info(deployment).get("id") + model_name: Final = deployment["litellm_params"].get("model") + if not isinstance(model_id, str) or model_name is None: + return None + try: + return self.get_deployment_model_info(model_id=model_id, model_name=model_name) + except Exception: + return None + + def _group_deployment(self, deployment: DeploymentTypedDict, team_id: str | None) -> _GroupDeployment: + model_info: Final = self._deployment_model_info(deployment) + return _GroupDeployment( + model_info=model_info, + price_entry=_deployment_price_entry(deployment, model_info), + team_may_use=team_may_use_deployment(_configured_model_info(deployment).get("team_id"), team_id), + ) def get_model_group_info( self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index 60c9e596588..29d844da0fd 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -4,9 +4,12 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc import datetime import enum +import math import re from collections.abc import Container, Mapping, Sequence from dataclasses import dataclass, field +from functools import lru_cache +from types import MappingProxyType from typing import ( TYPE_CHECKING, Annotated, @@ -23,13 +26,24 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx from openai import Timeout as SDKTimeout -from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator +from pydantic import ( + BeforeValidator, + ConfigDict, + Field, + JsonValue, + StrictInt, + TypeAdapter, + field_validator, + model_validator, + with_config, +) from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap as _GetModelCostMap from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.router_weights import RouterWeights @@ -60,6 +74,9 @@ from .utils import ( ModelResponse, StandardLoggingRoutingDecision, ) +from .utils import ( + is_server_derived_pricing_key as _is_server_derived_pricing_key, +) from .utils import ( server_owned_wif_litellm_params as _server_owned_wif_litellm_params, ) @@ -989,7 +1006,51 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]: return get_type_hints(model_class) +_RAW_FIELDS: Final = TypeAdapter(Mapping[str, object]) + + +def price_as_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return None + try: + price: Final = float(value) + except (ValueError, OverflowError): + return None + return price if math.isfinite(price) else None + + +def _outprices(price: float, listed: object) -> bool: + listed_price: Final = price_as_float(listed) + return listed_price is None or price > listed_price + + +@lru_cache(maxsize=4096) +def _is_usd_price_key_in(key: str, catalog_fields: frozenset[str]) -> bool: + return ( + ("cost_per_" in key or "token_cost" in key) + and "_dbu_" not in key + and (key in catalog_fields or _is_server_derived_pricing_key(key)) + ) + + +def is_usd_price_key(key: str) -> bool: + return _is_usd_price_key_in(key, _GetModelCostMap.loaded_model_cost_map_fields()) + + +def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]: + candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if is_usd_price_key(key)) + return MappingProxyType({key: price for key, price in candidates if price is not None}) + + +@with_config(ConfigDict(extra="ignore")) +class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth): + pass + + class ModelGroupInfo(LiteLLMBaseModel): + model_config = ConfigDict(extra="allow", json_schema_extra={"additionalProperties": True}) + __pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict + model_group: str providers: list[str] max_input_tokens: float | None = None @@ -1015,7 +1076,7 @@ class ModelGroupInfo(LiteLLMBaseModel): supports_fast_mode: bool = Field(default=False) supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None) supported_openai_params: list[str] | None = Field(default=[]) - configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None + configurable_clientside_auth_params: Sequence[str | ModelGroupClientsideAuthParam] | None = None description: str | None = None supported_endpoints: tuple[str, ...] | None = None @@ -1025,6 +1086,23 @@ class ModelGroupInfo(LiteLLMBaseModel): data[field_name] = False super().__init__(**data) + @model_validator(mode="before") + @classmethod + def _keep_only_price_extras(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + fields: Final = _RAW_FIELDS.validate_python(data) + field_names: Final = cls.model_fields.keys() + declared: Final = { + key: value for key, value in fields.items() if key in field_names and not is_usd_price_key(key) + } + return {**declared, **model_info_prices(fields)} + + def with_higher_prices(self, prices: Mapping[str, float]) -> "ModelGroupInfo": + listed: Final[Mapping[str, object]] = {**vars(self), **self.__pydantic_extra__} + higher: Final = {key: price for key, price in prices.items() if _outprices(price, listed.get(key))} + return self.model_copy(update=higher) if higher else self + class AssistantsTypedDict(TypedDict): custom_llm_provider: Literal["azure", "openai"] diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index c7781819ae4..865020ae1d2 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -176,6 +176,14 @@ ignored_function_names = [ "_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py "_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py + "_deployment_price_entry", # Read by _group_deployment, tested through the cost map model group info test in test_router.py + "_billed_pricing_entry", # Read by _deployment_price_entry, tested through the custom-priced model group info test in test_router.py + "listed_prices", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py + "_highest_billed_rate", # Read by _highest_billed_prices, tested through the tier and batch model group info test in test_router.py + "_highest_billed_prices", # Read by _set_model_group_info, tested through the tier and batch model group info test in test_router.py + "_is_model_group_member", # Read by _set_model_group_info, tested through every model group info test in test_router.py + "_deployment_model_info", # Read by _group_deployment, tested through the cost map model group info test in test_router.py + "_group_deployment", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py "_served_openai_models", # Cached as served_openai_models, tested through the pre-call check token count tests in test_router.py "priority_header_update", # Tested through the priority header tests in test_common_request_processing.py "_drops_params", # Read by split_request_priority and priority_header_update, tested through both diff --git a/tests/unit/proxy/proxy_server/test_routes_model_info.py b/tests/unit/proxy/proxy_server/test_routes_model_info.py index 9993361d930..68fb8f15abd 100644 --- a/tests/unit/proxy/proxy_server/test_routes_model_info.py +++ b/tests/unit/proxy/proxy_server/test_routes_model_info.py @@ -955,20 +955,9 @@ def _azure_group_deployment(sku_host: str, mode: str, **model_info: object) -> d } -@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)]) -def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment( - client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool -): - from litellm.proxy._types import LitellmUserRoles +def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyPatch) -> None: from litellm.proxy.agent_endpoints import model_list_helpers - router: Final = litellm.Router( - model_list=[ - _azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]), - _azure_group_deployment("batch-sku", "batch", team_id="team-a"), - ] - ) - async def append_agents_to_model_group(*, model_groups, user_api_key_dict): return model_groups @@ -981,6 +970,25 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group) ) + +@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)]) +def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment( + client: TestClient, + auth_as: Callable[..., AbstractContextManager[object]], + monkeypatch: pytest.MonkeyPatch, + team_id: str | None, + lists_batch: bool, +): + from litellm.proxy._types import LitellmUserRoles + + router: Final = litellm.Router( + model_list=[ + _azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]), + _azure_group_deployment("batch-sku", "batch", team_id="team-a"), + ] + ) + _serve_model_groups_from(router, monkeypatch) + with auth_as(LitellmUserRoles.PROXY_ADMIN, team_id=team_id): response: Final = client.get("/model_group/info") @@ -989,6 +997,102 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch +def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price( + client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy._types import LitellmUserRoles + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "image-pricing-group", + "litellm_params": {"model": "openai/made-up-image-model-a", "api_key": "dummy-key"}, + "model_info": { + "id": "image-deployment-a", + "output_cost_per_image": 0.5, + "output_cost_per_image_token": 2e-05, + "input_cost_per_token_priority": 6e-06, + "output_cost_per_image_4K": 0.16, + "output_cost_per_image_2K": "Infinity", + "input_cost_per_token": "NaN", + "custom_tags": ["images"], + "cost_center": 4021, + "input_dbu_cost_per_token": 7e-07, + }, + }, + { + "model_name": "image-pricing-group", + "litellm_params": {"model": "openai/made-up-image-model-b", "api_key": "dummy-key"}, + "model_info": { + "id": "image-deployment-b", + "output_cost_per_image": 0.25, + "output_cost_per_image_token": 4e-05, + "input_cost_per_token_above_272k_tokens": 8e-06, + }, + }, + ] + ) + _serve_model_groups_from(router, monkeypatch) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200, response.text + (group,) = response.json()["data"] + assert group["custom_tags"] == ["images"] + assert { + key: group.get(key) + for key in ( + "output_cost_per_image", + "output_cost_per_image_token", + "input_cost_per_token_priority", + "input_cost_per_token_above_272k_tokens", + "output_cost_per_image_4K", + "output_cost_per_image_2K", + ) + } == { + "output_cost_per_image": 0.5, + "output_cost_per_image_token": 4e-05, + "input_cost_per_token_priority": 6e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "output_cost_per_image_4K": 0.16, + "output_cost_per_image_2K": None, + } + assert group["input_cost_per_token"] is None + assert (group.get("cost_center"), group.get("input_dbu_cost_per_token")) == (None, None) + + +def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param( + client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy._types import LitellmUserRoles + + router: Final = litellm.Router( + model_list=[ + { + "model_name": "clientside-auth-group", + "litellm_params": { + "model": "openai/made-up-clientside-auth-model", + "api_key": "dummy-key", + "configurable_clientside_auth_params": [ + "api_key", + {"api_base": "^https://.*\\.example\\.com$", "unlisted_setting": "admin-only"}, + ], + }, + "model_info": {"id": "clientside-auth-deployment"}, + } + ] + ) + _serve_model_groups_from(router, monkeypatch) + + with auth_as(LitellmUserRoles.PROXY_ADMIN): + response: Final = client.get("/model_group/info") + + assert response.status_code == 200, response.text + (group,) = response.json()["data"] + assert group["configurable_clientside_auth_params"] == ["api_key", {"api_base": "^https://.*\\.example\\.com$"}] + + # --------------------------------------------------------------------------- # GET /v2/model/info?exclude_auto_routers # --------------------------------------------------------------------------- diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 035b7d9dcd3..fcd53b7b03f 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -221,6 +221,54 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None: assert math.isclose(after, before * 1000) +def _priced_group_router(extra_prices: dict[str, float], mode: str) -> Router: + return Router( + model_list=[ + { + "model_name": "priced-group", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "sk-fake", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + **extra_prices, + }, + "model_info": {"id": "priced-deployment", "mode": mode}, + } + ] + ) + + +@pytest.mark.parametrize( + "extra_prices, mode, route, body", + [ + ( + {"output_cost_per_reasoning_token": 1e-04}, + "chat", + "/chat/completions", + {"model": "priced-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10}, + ), + ( + {"output_cost_per_image": 0.5}, + "image_generation", + "/images/generations", + {"model": "priced-group", "prompt": "a cat", "n": 4}, + ), + ], +) +def test_prices_a_group_lists_leave_its_budget_reservation_unchanged( + extra_prices: dict[str, float], mode: str, route: str, body: dict[str, object] +) -> None: + without_extra: Final = estimate_request_max_cost( + request_body=body, route=route, llm_router=_priced_group_router({}, mode) + ) + with_extra: Final = estimate_request_max_cost( + request_body=body, route=route, llm_router=_priced_group_router(extra_prices, mode) + ) + + assert with_extra == without_extra + + @pytest.mark.asyncio @pytest.mark.parametrize( "expiry_offset, expected_max_budget", diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b55b3c5efc2..c1612540224 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -98,7 +98,7 @@ from litellm.llms.base_llm.vector_store.transformation import( LiteLLMVectorStoreEmbeddingExecutor, RouterVectorStoreEmbeddingExecutor, ) -from litellm.types.utils import CallTypes, CredentialItem, Usage +from litellm.types.utils import CallTypes, CredentialItem, ModelResponse, Usage from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE @@ -2525,6 +2525,410 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0 +@pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)]) +def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order: tuple[int, int]): + deployment_prices: Final = ( + { + "output_cost_per_image": 0.5, + "output_cost_per_image_token": 2e-05, + "input_cost_per_token_priority": 3e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + }, + { + "output_cost_per_image": 0.25, + "output_cost_per_image_token": 4e-05, + "input_cost_per_token_priority": 6e-06, + "input_cost_per_token_above_272k_tokens": 4e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_999k_tokens": 9e-06, + }, + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "tiered-pricing-group", + "litellm_params": {"model": f"openai/made-up-tiered-model-{index}", "api_key": "fake"}, + "model_info": { + "id": f"tiered-deployment-{index}", + **deployment_prices[index], + "output_cost_per_video_per_second": "free", + "output_cost_per_second": "inf", + "search_context_cost_per_query": {"search_context_size_low": 0.01}, + "input_dbu_cost_per_token": 7e-07, + "cost_center": 4021, + "cost_allocation_pct": "30", + "internal_chargeback_cost_per_token": 0.5, + "team_cost_per_seat": 40, + "input_cost_per_tokens": 9e-06, + }, + } + for index in deployment_order + ] + ) + + info: Final = router.get_model_group_info(model_group="tiered-pricing-group") + + assert info is not None + reported: Final = info.model_dump() + assert {key: reported.get(key) for key in deployment_prices[1]} == { + "output_cost_per_image": 0.5, + "output_cost_per_image_token": 4e-05, + "input_cost_per_token_priority": 6e-06, + "input_cost_per_token_above_272k_tokens": 8e-06, + "input_cost_per_token_batches": 1e-06, + "input_cost_per_token_above_999k_tokens": 9e-06, + } + assert reported.keys().isdisjoint( + { + "internal_chargeback_cost_per_token", + "team_cost_per_seat", + "input_cost_per_tokens", + "output_cost_per_video_per_second", + "output_cost_per_second", + "search_context_cost_per_query", + "input_dbu_cost_per_token", + "cost_center", + "cost_allocation_pct", + "litellm_provider", + "key", + "id", + } + ) + + +def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(monkeypatch: pytest.MonkeyPatch): + from litellm.utils import _invalidate_model_cost_lowercase_map + + cost_map_prices: Final = { + "output_cost_per_image": 0.08, + "output_cost_per_image_0.5K": 0.06, + "output_cost_per_image_4K": 0.16, + } + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + "made-up-resolution-model": { + "litellm_provider": "openai", + "mode": "image_generation", + **cost_map_prices, + "input_dbu_cost_per_token": 0.0002, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "resolution-group", + "litellm_params": {"model": "openai/made-up-resolution-model", "api_key": "fake"}, + } + ] + ) + + info: Final = router.get_model_group_info(model_group="resolution-group") + + assert info is not None + reported: Final = info.model_dump() + assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices + assert "input_dbu_cost_per_token" not in reported + + +def test_model_group_info_never_takes_prices_from_a_cost_map_entry_named_like_the_group( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.utils import _invalidate_model_cost_lowercase_map + + group_name: Final = "made-up-group-named-like-a-catalog-model" + router: Final = litellm.Router( + model_list=[ + { + "model_name": group_name, + "litellm_params": {"model": "gemini/made-up-unmapped-model", "api_key": "fake"}, + "model_info": {"id": "self-hosted-deployment", "input_cost_per_token": 1e-07}, + } + ] + ) + monkeypatch.setattr( + litellm, + "model_cost", + { + group_name: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 5e-06, + "input_cost_per_token_priority": 1e-05, + "output_cost_per_token_batches": 7e-06, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + + info: Final = router.get_model_group_info(model_group=group_name) + + assert info is not None + reported: Final = info.model_dump() + assert reported["input_cost_per_token"] == 1e-07 + assert reported.keys().isdisjoint({"input_cost_per_token_priority", "output_cost_per_token_batches"}) + + +def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_bills( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.utils import _invalidate_model_cost_lowercase_map + + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + "made-up-long-context-model": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-06, + "output_cost_per_token": 8e-06, + "input_cost_per_token_above_200k_tokens": 4e-06, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "custom-priced-group", + "litellm_params": { + "model": "openai/made-up-long-context-model", + "api_key": "fake", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 3e-06, + }, + "model_info": {"id": "custom-priced-deployment"}, + } + ] + ) + long_prompt_tokens: Final = 300_000 + + response: Final = router.completion( + model="custom-priced-group", + messages=[{"role": "user", "content": "hi"}], + mock_response=ModelResponse( + model="made-up-long-context-model", + choices=[{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop", "index": 0}], + usage=Usage(prompt_tokens=long_prompt_tokens, completion_tokens=0, total_tokens=long_prompt_tokens), + ), + ) + info: Final = router.get_model_group_info(model_group="custom-priced-group") + + assert info is not None + listed: Final = info.model_dump() + assert response._hidden_params["response_cost"] == pytest.approx(long_prompt_tokens * listed["input_cost_per_token"]) + assert "input_cost_per_token_above_200k_tokens" not in listed + + +def test_model_group_info_lists_tier_and_batch_prices_at_the_most_any_deployment_bills( + monkeypatch: pytest.MonkeyPatch, +): + from litellm.cost_calculator import batch_cost_calculator + from litellm.utils import _invalidate_model_cost_lowercase_map + + backends: Final = ("made-up-tiered-backend", "made-up-untiered-backend") + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + backends[0]: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "input_cost_per_token_flex": 5e-07, + "output_cost_per_token_flex": 2e-06, + "input_cost_per_token_batches": 4e-07, + "output_cost_per_token_batches": 2e-06, + }, + backends[1]: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.2e-05, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "mixed-backend-group", + "litellm_params": {"model": f"openai/{backend}", "api_key": "fake"}, + "model_info": {"id": backend}, + } + for backend in backends + ] + ) + tokens: Final = 1_000_000 + usage: Final = Usage(prompt_tokens=tokens, completion_tokens=tokens, total_tokens=2 * tokens) + flex_bills: Final = tuple( + litellm.cost_per_token( + model=backend, + custom_llm_provider="openai", + prompt_tokens=tokens, + completion_tokens=tokens, + service_tier="flex", + ) + for backend in backends + ) + batch_bills: Final = tuple( + batch_cost_calculator(usage, model=backend, custom_llm_provider="openai") for backend in backends + ) + highest_bills: Final = ( + max(prompt_cost for prompt_cost, _ in flex_bills), + max(completion_cost for _, completion_cost in flex_bills), + max(prompt_cost for prompt_cost, _ in batch_bills), + max(completion_cost for _, completion_cost in batch_bills), + ) + + info: Final = router.get_model_group_info(model_group="mixed-backend-group") + + assert info is not None + listed: Final = info.model_dump() + listed_keys: Final = ( + "input_cost_per_token_flex", + "output_cost_per_token_flex", + "input_cost_per_token_batches", + "output_cost_per_token_batches", + ) + assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills) + + +def test_model_group_info_lists_cache_prices_at_the_most_any_deployment_bills(monkeypatch: pytest.MonkeyPatch): + from litellm.utils import _invalidate_model_cost_lowercase_map + + backends: Final = ("made-up-cached-backend", "made-up-uncached-backend") + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + backends[0]: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "input_cost_per_token_priority": 2e-06, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_priority": 2e-07, + "cache_creation_input_token_cost": 1.25e-06, + }, + backends[1]: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.2e-05, + "input_cost_per_token_priority": 6e-06, + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "mixed-cache-group", + "litellm_params": {"model": f"openai/{backend}", "api_key": "fake"}, + "model_info": {"id": backend}, + } + for backend in backends + ] + ) + tokens: Final = 1_000_000 + + def highest_prompt_bill(cache_read_tokens: int, cache_creation_tokens: int, service_tier: str | None) -> float: + return max( + litellm.cost_per_token( + model=backend, + custom_llm_provider="openai", + prompt_tokens=tokens, + completion_tokens=0, + cache_read_input_tokens=cache_read_tokens, + cache_creation_input_tokens=cache_creation_tokens, + service_tier=service_tier, + )[0] + for backend in backends + ) + + highest_bills: Final = ( + highest_prompt_bill(tokens, 0, None), + highest_prompt_bill(tokens, 0, "priority"), + highest_prompt_bill(0, tokens, None), + ) + + info: Final = router.get_model_group_info(model_group="mixed-cache-group") + + assert info is not None + listed: Final = info.model_dump() + listed_keys: Final = ( + "cache_read_input_token_cost", + "cache_read_input_token_cost_priority", + "cache_creation_input_token_cost", + ) + assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills) + + +@pytest.mark.parametrize(("team_id", "listed_cache_read_cost"), [(None, 1e-07), ("team-b", 1e-07), ("team-a", 9e-06)]) +def test_model_group_info_lists_a_team_deployment_price_only_to_that_team( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, listed_cache_read_cost: float +): + from litellm.utils import _invalidate_model_cost_lowercase_map + + cache_read_costs: Final = {"made-up-shared-backend": 1e-07, "made-up-team-backend": 9e-06} + monkeypatch.setattr( + litellm, + "model_cost", + { + **litellm.model_cost, + **{ + backend: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-06, + "output_cost_per_token": 4e-06, + "cache_read_input_token_cost": cache_read_cost, + } + for backend, cache_read_cost in cache_read_costs.items() + }, + }, + ) + _invalidate_model_cost_lowercase_map() + router: Final = litellm.Router( + model_list=[ + { + "model_name": "team-scoped-group", + "litellm_params": {"model": "openai/made-up-shared-backend", "api_key": "fake"}, + "model_info": {"id": "shared-deployment"}, + }, + { + "model_name": "team-scoped-group", + "litellm_params": {"model": "openai/made-up-team-backend", "api_key": "fake"}, + "model_info": {"id": "team-a-deployment", "team_id": "team-a"}, + }, + ] + ) + + info: Final = router.get_model_group_info(model_group="team-scoped-group", team_id=team_id) + + assert info is not None + assert info.model_dump().get("cache_read_input_token_cost") == listed_cache_read_cost + + +def test_model_group_info_still_validates_from_attributes(): + source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06) + + info: Final = ModelGroupInfo.model_validate(source, from_attributes=True) + + assert (info.model_group, info.providers, info.input_cost_per_token) == ("attribute-group", ["openai"], 1e-06) + + def test_model_group_info_description_from_model_info(): """ model_info.description set in the config should surface on ModelGroupInfo, @@ -2870,12 +3274,18 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto (5, 5.0), (None, None), ("not-a-number", None), + (True, None), + ({"input_cost_per_token": 1e-05}, None), + ("Infinity", None), + ("-inf", None), + (float("nan"), None), + (10**400, None), ], ) -def test_cost_value_as_float(value, expected): - from litellm.router import _cost_value_as_float +def test_price_as_float(value: object, expected: float | None): + from litellm.types.router import price_as_float - assert _cost_value_as_float(value) == expected + assert price_as_float(value) == expected def test_model_group_info_with_stringified_cost_values(): diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx new file mode 100644 index 00000000000..622ad6c9e21 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx @@ -0,0 +1,43 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { ModelExtraPrices } from "./ModelExtraPrices"; + +describe("ModelExtraPrices", () => { + it("lists every USD price besides the per-token pair, sorted by name, with token prices per 1M tokens", () => { + render( + , + ); + const terms = screen.getAllByRole("term").map((term) => term.textContent); + const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent); + expect(terms.map((term, index) => [term, definitions[index]])).toEqual([ + ["cache_read_input_token_cost", "$0.5 per 1M tokens"], + ["input_cost_per_token_priority", "$2.5 per 1M tokens"], + ["output_cost_per_image", "$0.04"], + ["output_cost_per_image_4K", "$0.16"], + ]); + }); + + it("renders nothing when the model lists only the per-token prices", () => { + const { container } = render( + , + ); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx new file mode 100644 index 00000000000..cbecc76149e --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx @@ -0,0 +1,44 @@ +const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]); + +const TOKENS_PER_LISTED_PRICE = 1_000_000; + +const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 }); + +const isUsdPriceKey = (key: string) => + (key.includes("cost_per_") || key.includes("token_cost")) && !key.includes("_dbu_"); + +const isPerTokenPrice = (key: string) => /_token(_|$)/.test(key); + +const isPriceKey = (key: string) => isUsdPriceKey(key) && !PRICES_SHOWN_ELSEWHERE.has(key); + +const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value); + +const isExtraPrice = (entry: [string, unknown]): entry is [string, number] => + isPriceKey(entry[0]) && isFinitePrice(entry[1]); + +const formatPrice = (key: string, price: number) => + isPerTokenPrice(key) + ? `$${priceFormat.format(price * TOKENS_PER_LISTED_PRICE)} per 1M tokens` + : `$${priceFormat.format(price)}`; + +export function ModelExtraPrices({ model }: { model: Record }) { + const prices = Object.entries(model) + .filter(isExtraPrice) + .sort(([a], [b]) => a.localeCompare(b)); + if (prices.length === 0) { + return null; + } + return ( +
+

Other Prices (USD):

+
+ {prices.map(([key, price]) => ( +
+
{key}
+
{formatPrice(key, price)}
+
+ ))} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index 59f26b72481..ab7156256c1 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -56,6 +56,7 @@ import { } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; import { ModelCustomMetadata } from "./ModelCustomMetadata"; +import { ModelExtraPrices } from "./ModelExtraPrices"; interface ModelHubTableProps { accessToken: string | null; @@ -788,6 +789,7 @@ const ModelHubTable: React.FC = ({ : "Not specified"}

+ diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 59da41e7c05..1aaf7760ef2 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -55,6 +55,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; +import { ModelExtraPrices } from "@/components/AIHub/ModelExtraPrices"; import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders"; interface PublicModelHubProps { @@ -884,6 +885,7 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded : "Not specified"}

+ diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f114c1c73d2..c919c57ef50 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30543,17 +30543,12 @@ export interface components { worker_registry?: components["schemas"]["WorkerRegistryEntry"][] | null; }; /** ConfigurableClientsideParamsCustomAuth */ - "ConfigurableClientsideParamsCustomAuth-Input": { + ConfigurableClientsideParamsCustomAuth: { /** Api Base */ api_base: string; } & { [key: string]: unknown; }; - /** ConfigurableClientsideParamsCustomAuth */ - "ConfigurableClientsideParamsCustomAuth-Output": { - /** Api Base */ - api_base: string; - }; /** ContainerAuto */ ContainerAuto: { /** File Ids */ @@ -35412,7 +35407,7 @@ export interface components { /** Complexity Router Default Model */ complexity_router_default_model?: string | null; /** Configurable Clientside Auth Params */ - configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Input"])[] | null; + configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth"])[] | null; /** Cost Per Second */ cost_per_second?: number | null; /** Custom Llm Provider */ @@ -38490,10 +38485,15 @@ export interface components { */ warn_within_days: number; }; + /** ModelGroupClientsideAuthParam */ + ModelGroupClientsideAuthParam: { + /** Api Base */ + api_base: string; + }; /** ModelGroupInfoProxy */ ModelGroupInfoProxy: { /** Configurable Clientside Auth Params */ - configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null; + configurable_clientside_auth_params?: (string | components["schemas"]["ModelGroupClientsideAuthParam"])[] | null; /** Custom Metadata */ custom_metadata?: { [key: string]: string | number | boolean; @@ -38584,6 +38584,8 @@ export interface components { supports_web_search: boolean; /** Tpm */ tpm?: number | null; + } & { + [key: string]: unknown; }; /** ModelInfoDelete */ ModelInfoDelete: { @@ -50815,7 +50817,7 @@ export interface components { /** Complexity Router Default Model */ complexity_router_default_model?: string | null; /** Configurable Clientside Auth Params */ - configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Input"])[] | null; + configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth"])[] | null; /** Cost Per Second */ cost_per_second?: number | null; /** Custom Llm Provider */