scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-06 07:48:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 112

Merge 9f7fc215f6 into 67ae3fb527

Browse source
This commit is contained in:
m6atlas 2026-10-29 11:52:25 +00:00 • committed by GitHub
commit 3c7319bc0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 915 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
litellm/cost_calculator.py
View file

@ -909,7 +909,7 @@ def _bedrock_pricing_info(
def cost_map_key_prices_anything(key: str) -> bool:
entry: Final = _raw_cost_map_entry(key)
entry: Final = raw_cost_map_entry(key)
return entry is not None and _cost_map_entry_prices_anything(entry)
@ -2351,7 +2351,7 @@ def _deployment_model_info(
) -> ModelInfo | None:
if not custom_pricing:
return None
registered_deployment_info: Final = _raw_cost_map_entry(router_model_id) if router_model_id is not None else None
registered_deployment_info: Final = raw_cost_map_entry(router_model_id) if router_model_id is not None else None
if registered_deployment_info is not None:
return cast(ModelInfo, registered_deployment_info) # cast-ok: router registers deployment prices under its id
if litellm_logging_obj is None:
@ -2424,7 +2424,7 @@ def _priced_candidates_before_unpriced_rules(candidates: tuple[_CostCandidate, .
)
def _raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
def raw_cost_map_entry(key: str) -> Mapping[str, object] | None:
raw_entry: Final = litellm.model_cost.get(key)
return raw_entry if isinstance(raw_entry, Mapping) else None
@ -2468,7 +2468,7 @@ def pricing_entry_for_cost_calc(
)
if resolved is None:
return None
return resolved["key"], _raw_cost_map_entry(resolved["key"]) or resolved
return resolved["key"], raw_cost_map_entry(resolved["key"]) or resolved
def ocr_cost(

31
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -910,7 +910,9 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
return 0.0
def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
def get_cost_per_unit(
model_info: Mapping[str, object], cost_key: str, default_value: float | None = 0.0
) -> float | None:
# Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
cost_per_unit: Final = model_info.get(cost_key)
if isinstance(cost_per_unit, float):
@ -954,6 +956,33 @@ def get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float
_get_cost_per_unit = get_cost_per_unit
_HALF_PRICED_BATCH_KEYS: Final = frozenset(
{f"input_cost_per_token{_BATCH_KEY_SUFFIX}", f"output_cost_per_token{_BATCH_KEY_SUFFIX}"}
)
_TIER_KEY_SUFFIXES: Final = ("", *SERVICE_TIER_COST_KEY_SUFFIXES)
_CACHE_KEYS_BILLED_AT_INPUT_RATE: Final = MappingProxyType(
{
**{f"cache_read_input_token_cost{suffix}": f"input_cost_per_token{suffix}" for suffix in _TIER_KEY_SUFFIXES},
**{
f"cache_creation_input_token_cost{suffix}": f"input_cost_per_token{suffix}" for suffix in _TIER_KEY_SUFFIXES
},
}
)
def billed_rate(model_info: Mapping[str, object], cost_key: str) -> float | None:
rate: Final = get_cost_per_unit(model_info, cost_key, default_value=None)
if rate is not None:
return rate
if cost_key in _HALF_PRICED_BATCH_KEYS:
standard_rate: Final = get_cost_per_unit(
model_info, cost_key.removesuffix(_BATCH_KEY_SUFFIX), default_value=None
)
return None if standard_rate is None else standard_rate / 2
input_key: Final = _CACHE_KEYS_BILLED_AT_INPUT_RATE.get(cost_key)
return None if input_key is None else get_cost_per_unit(model_info, input_key, default_value=None)
def deployment_pricing(model_info: ModelInfo | None) -> ModelInfo | None:
"""The prices a deployment sets itself, as floats; None when it sets none that parse."""
if model_info is None:

6
litellm/proxy/auth/auth_utils.py
View file

@ -2,7 +2,7 @@ import importlib.util
import os
import re
import sys
from collections.abc import Collection, Iterator, Mapping
from collections.abc import Collection, Iterator, Mapping, Sequence
from functools import lru_cache
from itertools import accumulate, chain, repeat
from logging import Logger
@ -38,7 +38,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
)
from litellm.types.router import (
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
ConfigurableClientsideParamsCustomAuth,
Deployment,
reject_server_owned_wif_fields,
server_owned_wif_fields_present,
@ -189,7 +189,7 @@ def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool:
def _is_param_allowed(
param: str,
request_body_value: object,
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
configurable_clientside_auth_params: Sequence[str | ConfigurableClientsideParamsCustomAuth] | None,
) -> bool:
"""
Check if param is a str or dict and if request_body_value is in the list of allowed values

16
litellm/proxy/spend_tracking/budget_reservation.py
View file

@ -45,7 +45,7 @@ from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncreme
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.router import Router
from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
from litellm.types.router import DeploymentTypedDict
from litellm.types.router import DeploymentTypedDict, ModelGroupInfo
@dataclass
@ -706,14 +706,7 @@ def _model_access_group_counter(group: str, budget: ModelAccessGroupBudget | Non
def _dedupe_tags(tags: list[str]) -> list[str]:
seen: Final = set()
deduped_tags: Final = []
for tag in tags:
if tag in seen:
continue
seen.add(tag)
deduped_tags.append(tag)
return deduped_tags
return list(dict.fromkeys(tags))
@with_service_target(AUTH_OBJECTS_TARGET)
@ -1507,6 +1500,9 @@ def _estimate_image_generation_cost(
return cost_per_image * max(n, 1)
_DECLARED_GROUP_FIELDS: Final = MappingProxyType(dict.fromkeys(ModelGroupInfo.model_fields, True))
def _get_model_cost_info(
model: str,
llm_router: Router | None,
@ -1514,7 +1510,7 @@ def _get_model_cost_info(
if llm_router is not None:
model_group_info: Final = llm_router.cached_model_group_info(model)
if model_group_info is not None:
return model_group_info.model_dump()
return model_group_info.model_dump(include=_DECLARED_GROUP_FIELDS)
return dict(litellm.get_model_info(model=model))

152
litellm/router.py
View file

@ -103,6 +103,7 @@ from litellm.constants import (
RUNTIME_UPDATABLE_ROUTER_SETTINGS,
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
)
from litellm.cost_calculator import raw_cost_map_entry
from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
@ -142,7 +143,8 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
from litellm.litellm_core_utils.llm_cost_calc.utils import SERVICE_TIER_COST_KEY_SUFFIXES, billed_rate
from litellm.litellm_core_utils.ptu_pricing import (
PTU_COST_ATTRIBUTION_ENV_VAR,
declares_ptu,
@ -454,6 +456,9 @@ from litellm.types.router import (
SearchToolTypedDict,
TaggedPreRoutingStrategy,
holds_secret_pointer,
is_usd_price_key,
model_info_prices,
price_as_float,
)
from litellm.types.services import ServiceTypes
from litellm.types.utils import (
@ -547,15 +552,6 @@ RouterStrategySelector: TypeAlias = (
)
def _cost_value_as_float(value: str | float | None) -> float | None:
if value is None:
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def model_info_is_active_for_environment(model_info: Mapping[str, object] | None) -> bool:
"""Single owner of the environment-gating rule: a deployment whose model_info names
`supported_environments` loads only on pods whose LITELLM_ENVIRONMENT is in that list.
@ -724,6 +720,57 @@ def _configured_model_info(deployment: Mapping[str, object]) -> Mapping[str, obj
return _MODEL_INFO_ADAPTER.validate_python(deployment.get("model_info") or {})
def _billed_pricing_entry(deployment: Mapping[str, object], model_info: Mapping[str, object]) -> Mapping[str, object]:
configured_info: Final = _configured_model_info(deployment)
model_id: Final = configured_info.get("id")
custom_priced: Final = use_custom_pricing_for_model(
{
**_LITELLM_PARAMS_ADAPTER.validate_python(deployment.get("litellm_params") or {}),
"metadata": {"model_info": dict(configured_info)},
}
)
registered_entry: Final = raw_cost_map_entry(model_id) if custom_priced and isinstance(model_id, str) else None
if registered_entry is not None:
return registered_entry
cost_map_key: Final = model_info.get("key")
catalog_entry: Final = raw_cost_map_entry(cost_map_key) if isinstance(cost_map_key, str) else None
return model_info if catalog_entry is None else {**catalog_entry, **model_info}
def _deployment_price_entry(
deployment: Mapping[str, object], model_info: Mapping[str, object] | None
) -> Mapping[str, object]:
if model_info is None:
return _configured_model_info(deployment)
token_prices: Final = {key: model_info.get(key) for key in ("input_cost_per_token", "output_cost_per_token")}
return {**_billed_pricing_entry(deployment, model_info), **token_prices}
@dataclass(frozen=True, slots=True)
class _GroupDeployment:
model_info: ModelMapInfo | None
price_entry: Mapping[str, object]
team_may_use: bool
def listed_prices(self) -> Mapping[str, float]:
prices: Final = model_info_prices(self.price_entry)
if self.team_may_use:
return prices
return MappingProxyType({key: price for key, price in prices.items() if key in ModelGroupInfo.model_fields})
def _highest_billed_rate(cost_key: str, price_entries: Sequence[Mapping[str, object]]) -> float | None:
rates: Final = (price_as_float(billed_rate(entry, cost_key)) for entry in price_entries)
return max((rate for rate in rates if rate is not None), default=None)
def _highest_billed_prices(
listed_keys: Iterable[str], price_entries: Sequence[Mapping[str, object]]
) -> Mapping[str, float]:
rates: Final = ((key, _highest_billed_rate(key, price_entries)) for key in listed_keys)
return MappingProxyType({key: rate for key, rate in rates if rate is not None})
def _search_tool_support(deployment: DeploymentTypedDict) -> bool | None:
from litellm.responses.litellm_completion_transformation.tool_search import chat_config_builds_anthropic_request
@ -12676,17 +12723,12 @@ class Router:
(d for d in (_configured_model_info(m).get("description") for m in model_list) if isinstance(d, str)),
None,
)
for model in model_list:
is_match = False
if (
"model_name" in model
and model["model_name"] == model_group
or "model_name" in model
and self.pattern_router.route(model_group) is not None
): # exact match
is_match = True
if not is_match:
group_deployments: Final = tuple(
self._group_deployment(model, team_id) if self._is_model_group_member(model, model_group) else None
for model in model_list
)
for model, group_deployment in zip(model_list, group_deployments, strict=True):
if group_deployment is None:
continue
# model in model group found #
litellm_params = LiteLLM_Params(**model["litellm_params"])
@ -12727,17 +12769,9 @@ class Router:
if _deployment_otpm is None:
_deployment_otpm = model_info_dict.get("otpm", None)
# get model info
try:
model_id = model_info_dict.get("id", None)
if model_id is not None:
model_info = self.get_deployment_model_info(model_id=model_id, model_name=litellm_params.model)
else:
model_info = None
except Exception:
model_info = None
model_info = group_deployment.model_info
deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict)
deployment_prices = group_deployment.listed_prices()
# get llm provider
litellm_model, llm_provider = self._model_group_llm_provider(litellm_params)
@ -12752,8 +12786,8 @@ class Router:
# Get mode from database model_info if available, otherwise default to "chat"
db_model_info = model.get("model_info", {})
mode = db_model_info.get("mode", "chat")
input_cost_per_token = _cost_value_as_float(db_model_info.get("input_cost_per_token"))
output_cost_per_token = _cost_value_as_float(db_model_info.get("output_cost_per_token"))
input_cost_per_token = price_as_float(db_model_info.get("input_cost_per_token"))
output_cost_per_token = price_as_float(db_model_info.get("output_cost_per_token"))
model_info = ModelMapInfo(
key=model_group,
@ -12774,7 +12808,8 @@ class Router:
"model_group": user_facing_model_group_name,
"providers": [llm_provider],
"description": model_group_description,
**model_info,
**{key: value for key, value in model_info.items() if not is_usd_price_key(key)},
**deployment_prices,
"supports_fast_mode": True,
"supported_reasoning_efforts": None,
"supported_endpoints": None,
@ -12808,18 +12843,7 @@ class Router:
)
):
model_group_info.max_output_tokens = model_info["max_output_tokens"]
_input_cost_per_token = _cost_value_as_float(model_info.get("input_cost_per_token"))
if _input_cost_per_token is not None and (
model_group_info.input_cost_per_token is None
or _input_cost_per_token > (model_group_info.input_cost_per_token or 0.0)
):
model_group_info.input_cost_per_token = _input_cost_per_token
_output_cost_per_token = _cost_value_as_float(model_info.get("output_cost_per_token"))
if _output_cost_per_token is not None and (
model_group_info.output_cost_per_token is None
or _output_cost_per_token > (model_group_info.output_cost_per_token or 0.0)
):
model_group_info.output_cost_per_token = _output_cost_per_token
model_group_info = model_group_info.with_higher_prices(deployment_prices)
if (
model_info.get("supports_parallel_function_calling", None) is not None
and model_info["supports_parallel_function_calling"] is True
@ -12864,7 +12888,7 @@ class Router:
deployment_supported_endpoints(
model_info.get("supported_endpoints"), model, llm_provider
)
if team_may_use_deployment(_configured_model_info(model).get("team_id"), team_id)
if group_deployment.team_may_use
else ()
),
)
@ -12928,7 +12952,39 @@ class Router:
if configurable_clientside_auth_params is not None:
model_group_info.configurable_clientside_auth_params = configurable_clientside_auth_params
return model_group_info
usable_price_entries: Final = tuple(
deployment.price_entry
for deployment in group_deployments
if deployment is not None and deployment.team_may_use
)
return model_group_info.with_higher_prices(
_highest_billed_prices(model_info_prices(model_group_info.model_dump()), usable_price_entries)
)
return None
def _is_model_group_member(self, deployment: DeploymentTypedDict, model_group: str) -> bool:
return "model_name" in deployment and (
deployment["model_name"] == model_group or self.pattern_router.route(model_group) is not None
)
def _deployment_model_info(self, deployment: DeploymentTypedDict) -> ModelMapInfo | None:
model_id: Final = _configured_model_info(deployment).get("id")
model_name: Final = deployment["litellm_params"].get("model")
if not isinstance(model_id, str) or model_name is None:
return None
try:
return self.get_deployment_model_info(model_id=model_id, model_name=model_name)
except Exception:
return None
def _group_deployment(self, deployment: DeploymentTypedDict, team_id: str | None) -> _GroupDeployment:
model_info: Final = self._deployment_model_info(deployment)
return _GroupDeployment(
model_info=model_info,
price_entry=_deployment_price_entry(deployment, model_info),
team_may_use=team_may_use_deployment(_configured_model_info(deployment).get("team_id"), team_id),
)
def get_model_group_info(
self, model_group: str, *, include_hidden: bool = False, team_id: str | None = None

82
litellm/types/router.py
View file

@ -4,9 +4,12 @@ litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
import datetime
import enum
import math
import re
from collections.abc import Container, Mapping, Sequence
from dataclasses import dataclass, field
from functools import lru_cache
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
Annotated,
@ -23,13 +26,24 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import httpx
from openai import Timeout as SDKTimeout
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
from pydantic import (
BeforeValidator,
ConfigDict,
Field,
JsonValue,
StrictInt,
TypeAdapter,
field_validator,
model_validator,
with_config,
)
from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap as _GetModelCostMap
from litellm.litellm_core_utils.provider_affinity import validate_provider_affinity_header_name
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router_weights import RouterWeights
@ -60,6 +74,9 @@ from .utils import (
ModelResponse,
StandardLoggingRoutingDecision,
)
from .utils import (
is_server_derived_pricing_key as _is_server_derived_pricing_key,
)
from .utils import (
server_owned_wif_litellm_params as _server_owned_wif_litellm_params,
)
@ -989,7 +1006,51 @@ def _resolved_annotations(model_class: type[object]) -> Mapping[str, object]:
return get_type_hints(model_class)
_RAW_FIELDS: Final = TypeAdapter(Mapping[str, object])
def price_as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, int | float | str):
return None
try:
price: Final = float(value)
except (ValueError, OverflowError):
return None
return price if math.isfinite(price) else None
def _outprices(price: float, listed: object) -> bool:
listed_price: Final = price_as_float(listed)
return listed_price is None or price > listed_price
@lru_cache(maxsize=4096)
def _is_usd_price_key_in(key: str, catalog_fields: frozenset[str]) -> bool:
return (
("cost_per_" in key or "token_cost" in key)
and "_dbu_" not in key
and (key in catalog_fields or _is_server_derived_pricing_key(key))
)
def is_usd_price_key(key: str) -> bool:
return _is_usd_price_key_in(key, _GetModelCostMap.loaded_model_cost_map_fields())
def model_info_prices(model_info: Mapping[str, object]) -> Mapping[str, float]:
candidates: Final = ((key, price_as_float(value)) for key, value in model_info.items() if is_usd_price_key(key))
return MappingProxyType({key: price for key, price in candidates if price is not None})
@with_config(ConfigDict(extra="ignore"))
class ModelGroupClientsideAuthParam(ConfigurableClientsideParamsCustomAuth):
pass
class ModelGroupInfo(LiteLLMBaseModel):
model_config = ConfigDict(extra="allow", json_schema_extra={"additionalProperties": True})
__pydantic_extra__: dict[str, float] = Field(init=False) # pyright: ignore[reportIncompatibleVariableOverride] # pydantic's documented way to type extras # mutable-ok: pydantic only types extras via a dict
model_group: str
providers: list[str]
max_input_tokens: float | None = None
@ -1015,7 +1076,7 @@ class ModelGroupInfo(LiteLLMBaseModel):
supports_fast_mode: bool = Field(default=False)
supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None)
supported_openai_params: list[str] | None = Field(default=[])
configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
configurable_clientside_auth_params: Sequence[str | ModelGroupClientsideAuthParam] | None = None
description: str | None = None
supported_endpoints: tuple[str, ...] | None = None
@ -1025,6 +1086,23 @@ class ModelGroupInfo(LiteLLMBaseModel):
data[field_name] = False
super().__init__(**data)
@model_validator(mode="before")
@classmethod
def _keep_only_price_extras(cls, data: object) -> object:
if not isinstance(data, Mapping):
return data
fields: Final = _RAW_FIELDS.validate_python(data)
field_names: Final = cls.model_fields.keys()
declared: Final = {
key: value for key, value in fields.items() if key in field_names and not is_usd_price_key(key)
}
return {**declared, **model_info_prices(fields)}
def with_higher_prices(self, prices: Mapping[str, float]) -> "ModelGroupInfo":
listed: Final[Mapping[str, object]] = {**vars(self), **self.__pydantic_extra__}
higher: Final = {key: price for key, price in prices.items() if _outprices(price, listed.get(key))}
return self.model_copy(update=higher) if higher else self
class AssistantsTypedDict(TypedDict):
custom_llm_provider: Literal["azure", "openai"]

8
tests/code_coverage_tests/router_code_coverage.py
View file

@ -176,6 +176,14 @@ ignored_function_names = [
"_responses_continuation", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_responses_continuation_ceilings", # Tested through the Responses mid-stream fallback tests in tests/unit/llms/base_llm/responses/test_transformation.py
"_team_deployments_named", # Tested through the team lookups in the concurrent delete tests in test_router.py
"_deployment_price_entry", # Read by _group_deployment, tested through the cost map model group info test in test_router.py
"_billed_pricing_entry", # Read by _deployment_price_entry, tested through the custom-priced model group info test in test_router.py
"listed_prices", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py
"_highest_billed_rate", # Read by _highest_billed_prices, tested through the tier and batch model group info test in test_router.py
"_highest_billed_prices", # Read by _set_model_group_info, tested through the tier and batch model group info test in test_router.py
"_is_model_group_member", # Read by _set_model_group_info, tested through every model group info test in test_router.py
"_deployment_model_info", # Read by _group_deployment, tested through the cost map model group info test in test_router.py
"_group_deployment", # Read by _set_model_group_info, tested through the team-scoped model group info test in test_router.py
"_served_openai_models", # Cached as served_openai_models, tested through the pre-call check token count tests in test_router.py
"priority_header_update", # Tested through the priority header tests in test_common_request_processing.py
"_drops_params", # Read by split_request_priority and priority_header_update, tested through both

128
tests/unit/proxy/proxy_server/test_routes_model_info.py
View file

@ -955,20 +955,9 @@ def _azure_group_deployment(sku_host: str, mode: str, **model_info: object) -> d
}
@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
client, auth_as, monkeypatch, team_id: str | None, lists_batch: bool
):
from litellm.proxy._types import LitellmUserRoles
def _serve_model_groups_from(router: litellm.Router, monkeypatch: pytest.MonkeyPatch) -> None:
from litellm.proxy.agent_endpoints import model_list_helpers
router: Final = litellm.Router(
model_list=[
_azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]),
_azure_group_deployment("batch-sku", "batch", team_id="team-a"),
]
)
async def append_agents_to_model_group(*, model_groups, user_api_key_dict):
return model_groups
@ -981,6 +970,25 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
model_list_helpers, "append_agents_to_model_group", AsyncMock(side_effect=append_agents_to_model_group)
)
@pytest.mark.parametrize(("team_id", "lists_batch"), [(None, False), ("team-a", True)])
def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deployment(
client: TestClient,
auth_as: Callable[..., AbstractContextManager[object]],
monkeypatch: pytest.MonkeyPatch,
team_id: str | None,
lists_batch: bool,
):
from litellm.proxy._types import LitellmUserRoles
router: Final = litellm.Router(
model_list=[
_azure_group_deployment("chat-sku", "chat", supported_endpoints=["/v1/chat/completions"]),
_azure_group_deployment("batch-sku", "batch", team_id="team-a"),
]
)
_serve_model_groups_from(router, monkeypatch)
with auth_as(LitellmUserRoles.PROXY_ADMIN, team_id=team_id):
response: Final = client.get("/model_group/info")
@ -989,6 +997,102 @@ def test_model_group_info_lists_batch_only_to_the_team_that_owns_the_batch_deplo
assert ("/v1/batch" in (groups["gpt-mixed"]["supported_endpoints"] or ())) is lists_batch
def test_model_group_info_reports_every_price_the_proxy_bills_at_the_highest_deployment_price(
client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy._types import LitellmUserRoles
router: Final = litellm.Router(
model_list=[
{
"model_name": "image-pricing-group",
"litellm_params": {"model": "openai/made-up-image-model-a", "api_key": "dummy-key"},
"model_info": {
"id": "image-deployment-a",
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 2e-05,
"input_cost_per_token_priority": 6e-06,
"output_cost_per_image_4K": 0.16,
"output_cost_per_image_2K": "Infinity",
"input_cost_per_token": "NaN",
"custom_tags": ["images"],
"cost_center": 4021,
"input_dbu_cost_per_token": 7e-07,
},
},
{
"model_name": "image-pricing-group",
"litellm_params": {"model": "openai/made-up-image-model-b", "api_key": "dummy-key"},
"model_info": {
"id": "image-deployment-b",
"output_cost_per_image": 0.25,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_above_272k_tokens": 8e-06,
},
},
]
)
_serve_model_groups_from(router, monkeypatch)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response: Final = client.get("/model_group/info")
assert response.status_code == 200, response.text
(group,) = response.json()["data"]
assert group["custom_tags"] == ["images"]
assert {
key: group.get(key)
for key in (
"output_cost_per_image",
"output_cost_per_image_token",
"input_cost_per_token_priority",
"input_cost_per_token_above_272k_tokens",
"output_cost_per_image_4K",
"output_cost_per_image_2K",
)
} == {
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"output_cost_per_image_4K": 0.16,
"output_cost_per_image_2K": None,
}
assert group["input_cost_per_token"] is None
assert (group.get("cost_center"), group.get("input_dbu_cost_per_token")) == (None, None)
def test_model_group_info_lists_only_the_api_base_of_a_clientside_auth_param(
client: TestClient, auth_as: Callable[..., AbstractContextManager[object]], monkeypatch: pytest.MonkeyPatch
):
from litellm.proxy._types import LitellmUserRoles
router: Final = litellm.Router(
model_list=[
{
"model_name": "clientside-auth-group",
"litellm_params": {
"model": "openai/made-up-clientside-auth-model",
"api_key": "dummy-key",
"configurable_clientside_auth_params": [
"api_key",
{"api_base": "^https://.*\\.example\\.com$", "unlisted_setting": "admin-only"},
],
},
"model_info": {"id": "clientside-auth-deployment"},
}
]
)
_serve_model_groups_from(router, monkeypatch)
with auth_as(LitellmUserRoles.PROXY_ADMIN):
response: Final = client.get("/model_group/info")
assert response.status_code == 200, response.text
(group,) = response.json()["data"]
assert group["configurable_clientside_auth_params"] == ["api_key", {"api_base": "^https://.*\\.example\\.com$"}]
# ---------------------------------------------------------------------------
# GET /v2/model/info?exclude_auto_routers
# ---------------------------------------------------------------------------

48
tests/unit/proxy/spend_tracking/test_budget_reservation.py
View file

@ -221,6 +221,54 @@ def test_deployment_pricing_update_invalidates_cached_estimate() -> None:
assert math.isclose(after, before * 1000)
def _priced_group_router(extra_prices: dict[str, float], mode: str) -> Router:
return Router(
model_list=[
{
"model_name": "priced-group",
"litellm_params": {
"model": "openai/gpt-4o",
"api_key": "sk-fake",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 2e-06,
**extra_prices,
},
"model_info": {"id": "priced-deployment", "mode": mode},
}
]
)
@pytest.mark.parametrize(
"extra_prices, mode, route, body",
[
(
{"output_cost_per_reasoning_token": 1e-04},
"chat",
"/chat/completions",
{"model": "priced-group", "messages": [{"role": "user", "content": "hello"}], "max_tokens": 10},
),
(
{"output_cost_per_image": 0.5},
"image_generation",
"/images/generations",
{"model": "priced-group", "prompt": "a cat", "n": 4},
),
],
)
def test_prices_a_group_lists_leave_its_budget_reservation_unchanged(
extra_prices: dict[str, float], mode: str, route: str, body: dict[str, object]
) -> None:
without_extra: Final = estimate_request_max_cost(
request_body=body, route=route, llm_router=_priced_group_router({}, mode)
)
with_extra: Final = estimate_request_max_cost(
request_body=body, route=route, llm_router=_priced_group_router(extra_prices, mode)
)
assert with_extra == without_extra
@pytest.mark.asyncio
@pytest.mark.parametrize(
"expiry_offset, expected_max_budget",

418
tests/unit/test_router/test_router.py
View file

@ -98,7 +98,7 @@ from litellm.llms.base_llm.vector_store.transformation import(
LiteLLMVectorStoreEmbeddingExecutor,
RouterVectorStoreEmbeddingExecutor,
)
from litellm.types.utils import CallTypes, CredentialItem, Usage
from litellm.types.utils import CallTypes, CredentialItem, ModelResponse, Usage
from litellm.utils import _invalidate_model_cost_lowercase_map
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE
@ -2525,6 +2525,410 @@ def test_model_group_info_cost_none_for_unpriced_deployment_but_zero_when_declar
assert priced.output_cost_per_token is not None and priced.output_cost_per_token > 0
@pytest.mark.parametrize("deployment_order", [(0, 1), (1, 0)])
def test_model_group_info_reports_every_price_at_the_highest_across_deployments(deployment_order: tuple[int, int]):
deployment_prices: Final = (
{
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 2e-05,
"input_cost_per_token_priority": 3e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
},
{
"output_cost_per_image": 0.25,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_above_999k_tokens": 9e-06,
},
)
router: Final = litellm.Router(
model_list=[
{
"model_name": "tiered-pricing-group",
"litellm_params": {"model": f"openai/made-up-tiered-model-{index}", "api_key": "fake"},
"model_info": {
"id": f"tiered-deployment-{index}",
**deployment_prices[index],
"output_cost_per_video_per_second": "free",
"output_cost_per_second": "inf",
"search_context_cost_per_query": {"search_context_size_low": 0.01},
"input_dbu_cost_per_token": 7e-07,
"cost_center": 4021,
"cost_allocation_pct": "30",
"internal_chargeback_cost_per_token": 0.5,
"team_cost_per_seat": 40,
"input_cost_per_tokens": 9e-06,
},
}
for index in deployment_order
]
)
info: Final = router.get_model_group_info(model_group="tiered-pricing-group")
assert info is not None
reported: Final = info.model_dump()
assert {key: reported.get(key) for key in deployment_prices[1]} == {
"output_cost_per_image": 0.5,
"output_cost_per_image_token": 4e-05,
"input_cost_per_token_priority": 6e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_batches": 1e-06,
"input_cost_per_token_above_999k_tokens": 9e-06,
}
assert reported.keys().isdisjoint(
{
"internal_chargeback_cost_per_token",
"team_cost_per_seat",
"input_cost_per_tokens",
"output_cost_per_video_per_second",
"output_cost_per_second",
"search_context_cost_per_query",
"input_dbu_cost_per_token",
"cost_center",
"cost_allocation_pct",
"litellm_provider",
"key",
"id",
}
)
def test_model_group_info_lists_cost_map_prices_outside_the_typed_model_info(monkeypatch: pytest.MonkeyPatch):
from litellm.utils import _invalidate_model_cost_lowercase_map
cost_map_prices: Final = {
"output_cost_per_image": 0.08,
"output_cost_per_image_0.5K": 0.06,
"output_cost_per_image_4K": 0.16,
}
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
"made-up-resolution-model": {
"litellm_provider": "openai",
"mode": "image_generation",
**cost_map_prices,
"input_dbu_cost_per_token": 0.0002,
},
},
)
_invalidate_model_cost_lowercase_map()
router: Final = litellm.Router(
model_list=[
{
"model_name": "resolution-group",
"litellm_params": {"model": "openai/made-up-resolution-model", "api_key": "fake"},
}
]
)
info: Final = router.get_model_group_info(model_group="resolution-group")
assert info is not None
reported: Final = info.model_dump()
assert {key: reported.get(key) for key in cost_map_prices} == cost_map_prices
assert "input_dbu_cost_per_token" not in reported
def test_model_group_info_never_takes_prices_from_a_cost_map_entry_named_like_the_group(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.utils import _invalidate_model_cost_lowercase_map
group_name: Final = "made-up-group-named-like-a-catalog-model"
router: Final = litellm.Router(
model_list=[
{
"model_name": group_name,
"litellm_params": {"model": "gemini/made-up-unmapped-model", "api_key": "fake"},
"model_info": {"id": "self-hosted-deployment", "input_cost_per_token": 1e-07},
}
]
)
monkeypatch.setattr(
litellm,
"model_cost",
{
group_name: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 5e-06,
"input_cost_per_token_priority": 1e-05,
"output_cost_per_token_batches": 7e-06,
},
},
)
_invalidate_model_cost_lowercase_map()
info: Final = router.get_model_group_info(model_group=group_name)
assert info is not None
reported: Final = info.model_dump()
assert reported["input_cost_per_token"] == 1e-07
assert reported.keys().isdisjoint({"input_cost_per_token_priority", "output_cost_per_token_batches"})
def test_model_group_info_lists_a_custom_priced_deployment_only_at_the_rates_it_bills(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.utils import _invalidate_model_cost_lowercase_map
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
"made-up-long-context-model": {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 2e-06,
"output_cost_per_token": 8e-06,
"input_cost_per_token_above_200k_tokens": 4e-06,
},
},
)
_invalidate_model_cost_lowercase_map()
router: Final = litellm.Router(
model_list=[
{
"model_name": "custom-priced-group",
"litellm_params": {
"model": "openai/made-up-long-context-model",
"api_key": "fake",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 3e-06,
},
"model_info": {"id": "custom-priced-deployment"},
}
]
)
long_prompt_tokens: Final = 300_000
response: Final = router.completion(
model="custom-priced-group",
messages=[{"role": "user", "content": "hi"}],
mock_response=ModelResponse(
model="made-up-long-context-model",
choices=[{"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop", "index": 0}],
usage=Usage(prompt_tokens=long_prompt_tokens, completion_tokens=0, total_tokens=long_prompt_tokens),
),
)
info: Final = router.get_model_group_info(model_group="custom-priced-group")
assert info is not None
listed: Final = info.model_dump()
assert response._hidden_params["response_cost"] == pytest.approx(long_prompt_tokens * listed["input_cost_per_token"])
assert "input_cost_per_token_above_200k_tokens" not in listed
def test_model_group_info_lists_tier_and_batch_prices_at_the_most_any_deployment_bills(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.cost_calculator import batch_cost_calculator
from litellm.utils import _invalidate_model_cost_lowercase_map
backends: Final = ("made-up-tiered-backend", "made-up-untiered-backend")
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
backends[0]: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"input_cost_per_token_flex": 5e-07,
"output_cost_per_token_flex": 2e-06,
"input_cost_per_token_batches": 4e-07,
"output_cost_per_token_batches": 2e-06,
},
backends[1]: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.2e-05,
},
},
)
_invalidate_model_cost_lowercase_map()
router: Final = litellm.Router(
model_list=[
{
"model_name": "mixed-backend-group",
"litellm_params": {"model": f"openai/{backend}", "api_key": "fake"},
"model_info": {"id": backend},
}
for backend in backends
]
)
tokens: Final = 1_000_000
usage: Final = Usage(prompt_tokens=tokens, completion_tokens=tokens, total_tokens=2 * tokens)
flex_bills: Final = tuple(
litellm.cost_per_token(
model=backend,
custom_llm_provider="openai",
prompt_tokens=tokens,
completion_tokens=tokens,
service_tier="flex",
)
for backend in backends
)
batch_bills: Final = tuple(
batch_cost_calculator(usage, model=backend, custom_llm_provider="openai") for backend in backends
)
highest_bills: Final = (
max(prompt_cost for prompt_cost, _ in flex_bills),
max(completion_cost for _, completion_cost in flex_bills),
max(prompt_cost for prompt_cost, _ in batch_bills),
max(completion_cost for _, completion_cost in batch_bills),
)
info: Final = router.get_model_group_info(model_group="mixed-backend-group")
assert info is not None
listed: Final = info.model_dump()
listed_keys: Final = (
"input_cost_per_token_flex",
"output_cost_per_token_flex",
"input_cost_per_token_batches",
"output_cost_per_token_batches",
)
assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills)
def test_model_group_info_lists_cache_prices_at_the_most_any_deployment_bills(monkeypatch: pytest.MonkeyPatch):
from litellm.utils import _invalidate_model_cost_lowercase_map
backends: Final = ("made-up-cached-backend", "made-up-uncached-backend")
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
backends[0]: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"input_cost_per_token_priority": 2e-06,
"cache_read_input_token_cost": 1e-07,
"cache_read_input_token_cost_priority": 2e-07,
"cache_creation_input_token_cost": 1.25e-06,
},
backends[1]: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 3e-06,
"output_cost_per_token": 1.2e-05,
"input_cost_per_token_priority": 6e-06,
},
},
)
_invalidate_model_cost_lowercase_map()
router: Final = litellm.Router(
model_list=[
{
"model_name": "mixed-cache-group",
"litellm_params": {"model": f"openai/{backend}", "api_key": "fake"},
"model_info": {"id": backend},
}
for backend in backends
]
)
tokens: Final = 1_000_000
def highest_prompt_bill(cache_read_tokens: int, cache_creation_tokens: int, service_tier: str | None) -> float:
return max(
litellm.cost_per_token(
model=backend,
custom_llm_provider="openai",
prompt_tokens=tokens,
completion_tokens=0,
cache_read_input_tokens=cache_read_tokens,
cache_creation_input_tokens=cache_creation_tokens,
service_tier=service_tier,
)[0]
for backend in backends
)
highest_bills: Final = (
highest_prompt_bill(tokens, 0, None),
highest_prompt_bill(tokens, 0, "priority"),
highest_prompt_bill(0, tokens, None),
)
info: Final = router.get_model_group_info(model_group="mixed-cache-group")
assert info is not None
listed: Final = info.model_dump()
listed_keys: Final = (
"cache_read_input_token_cost",
"cache_read_input_token_cost_priority",
"cache_creation_input_token_cost",
)
assert tuple(listed[key] * tokens for key in listed_keys) == pytest.approx(highest_bills)
@pytest.mark.parametrize(("team_id", "listed_cache_read_cost"), [(None, 1e-07), ("team-b", 1e-07), ("team-a", 9e-06)])
def test_model_group_info_lists_a_team_deployment_price_only_to_that_team(
monkeypatch: pytest.MonkeyPatch, team_id: str | None, listed_cache_read_cost: float
):
from litellm.utils import _invalidate_model_cost_lowercase_map
cache_read_costs: Final = {"made-up-shared-backend": 1e-07, "made-up-team-backend": 9e-06}
monkeypatch.setattr(
litellm,
"model_cost",
{
**litellm.model_cost,
**{
backend: {
"litellm_provider": "openai",
"mode": "chat",
"input_cost_per_token": 1e-06,
"output_cost_per_token": 4e-06,
"cache_read_input_token_cost": cache_read_cost,
}
for backend, cache_read_cost in cache_read_costs.items()
},
},
)
_invalidate_model_cost_lowercase_map()
router: Final = litellm.Router(
model_list=[
{
"model_name": "team-scoped-group",
"litellm_params": {"model": "openai/made-up-shared-backend", "api_key": "fake"},
"model_info": {"id": "shared-deployment"},
},
{
"model_name": "team-scoped-group",
"litellm_params": {"model": "openai/made-up-team-backend", "api_key": "fake"},
"model_info": {"id": "team-a-deployment", "team_id": "team-a"},
},
]
)
info: Final = router.get_model_group_info(model_group="team-scoped-group", team_id=team_id)
assert info is not None
assert info.model_dump().get("cache_read_input_token_cost") == listed_cache_read_cost
def test_model_group_info_still_validates_from_attributes():
source: Final = SimpleNamespace(model_group="attribute-group", providers=["openai"], input_cost_per_token=1e-06)
info: Final = ModelGroupInfo.model_validate(source, from_attributes=True)
assert (info.model_group, info.providers, info.input_cost_per_token) == ("attribute-group", ["openai"], 1e-06)
def test_model_group_info_description_from_model_info():
"""
model_info.description set in the config should surface on ModelGroupInfo,
@ -2870,12 +3274,18 @@ def test_switch_routing_strategy_installs_lar1_then_restores_the_default_selecto
(5, 5.0),
(None, None),
("not-a-number", None),
(True, None),
({"input_cost_per_token": 1e-05}, None),
("Infinity", None),
("-inf", None),
(float("nan"), None),
(10**400, None),
],
)
def test_cost_value_as_float(value, expected):
from litellm.router import _cost_value_as_float
def test_price_as_float(value: object, expected: float | None):
from litellm.types.router import price_as_float
assert _cost_value_as_float(value) == expected
assert price_as_float(value) == expected
def test_model_group_info_with_stringified_cost_values():

43
ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.test.tsx Normal file
View file

@ -0,0 +1,43 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
import { ModelExtraPrices } from "./ModelExtraPrices";
describe("ModelExtraPrices", () => {
it("lists every USD price besides the per-token pair, sorted by name, with token prices per 1M tokens", () => {
render(
<ModelExtraPrices
model={{
model_group: "image-model",
input_cost_per_token: 0.000001,
output_cost_per_token: 0.000002,
output_cost_per_image: 0.04,
input_cost_per_token_priority: 0.0000025,
output_cost_per_image_4K: 0.16,
cache_read_input_token_cost: 0.0000005,
input_dbu_cost_per_token: 0.000142858,
cost_center: 4021,
output_cost_per_video_per_second: "free",
search_context_cost_per_query: { search_context_size_low: 0.01 },
max_input_tokens: 128000,
}}
/>,
);
const terms = screen.getAllByRole("term").map((term) => term.textContent);
const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent);
expect(terms.map((term, index) => [term, definitions[index]])).toEqual([
["cache_read_input_token_cost", "$0.5 per 1M tokens"],
["input_cost_per_token_priority", "$2.5 per 1M tokens"],
["output_cost_per_image", "$0.04"],
["output_cost_per_image_4K", "$0.16"],
]);
});
it("renders nothing when the model lists only the per-token prices", () => {
const { container } = render(
<ModelExtraPrices
model={{ model_group: "chat-model", input_cost_per_token: 0.000001, output_cost_per_token: 0.000002 }}
/>,
);
expect(container).toBeEmptyDOMElement();
});
});

44
ui/litellm-dashboard/src/components/AIHub/ModelExtraPrices.tsx Normal file
View file

@ -0,0 +1,44 @@
const PRICES_SHOWN_ELSEWHERE = new Set(["input_cost_per_token", "output_cost_per_token"]);
const TOKENS_PER_LISTED_PRICE = 1_000_000;
const priceFormat = new Intl.NumberFormat("en-US", { maximumSignificantDigits: 6 });
const isUsdPriceKey = (key: string) =>
(key.includes("cost_per_") || key.includes("token_cost")) && !key.includes("_dbu_");
const isPerTokenPrice = (key: string) => /_token(_|$)/.test(key);
const isPriceKey = (key: string) => isUsdPriceKey(key) && !PRICES_SHOWN_ELSEWHERE.has(key);
const isFinitePrice = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
const isExtraPrice = (entry: [string, unknown]): entry is [string, number] =>
isPriceKey(entry[0]) && isFinitePrice(entry[1]);
const formatPrice = (key: string, price: number) =>
isPerTokenPrice(key)
? `$${priceFormat.format(price * TOKENS_PER_LISTED_PRICE)} per 1M tokens`
: `$${priceFormat.format(price)}`;
export function ModelExtraPrices({ model }: { model: Record<string, unknown> }) {
const prices = Object.entries(model)
.filter(isExtraPrice)
.sort(([a], [b]) => a.localeCompare(b));
if (prices.length === 0) {
return null;
}
return (
<div className="col-span-2">
<p className="font-medium">Other Prices (USD):</p>
<dl className="mt-1 grid grid-cols-[max-content_1fr] gap-x-4 gap-y-1">
{prices.map(([key, price]) => (
<div key={key} className="contents">
<dt className="text-muted-foreground">{key}</dt>
<dd>{formatPrice(key, price)}</dd>
</div>
))}
</dl>
</div>
);
}

2
ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
View file

@ -56,6 +56,7 @@ import {
} from "@/lib/decisionModels";
import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden";
import { ModelCustomMetadata } from "./ModelCustomMetadata";
import { ModelExtraPrices } from "./ModelExtraPrices";
interface ModelHubTableProps {
accessToken: string | null;
@ -788,6 +789,7 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({
: "Not specified"}
</p>
</div>
<ModelExtraPrices model={selectedModel} />
</div>
</div>

2
ui/litellm-dashboard/src/components/public_model_hub.tsx
View file

@ -55,6 +55,7 @@ import { displayMode, isDecisionMode } from "@/lib/decisionModels";
import { modelUsageExample } from "@/components/AIHub/modelUsageExample";
import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils";
import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata";
import { ModelExtraPrices } from "@/components/AIHub/ModelExtraPrices";
import ModelHubProviders, { formatModelModeLabel } from "@/components/AIHub/ModelHubProviders";
interface PublicModelHubProps {
@ -884,6 +885,7 @@ const PublicModelHub: React.FC<PublicModelHubProps> = ({ accessToken, isEmbedded
: "Not specified"}
</p>
</div>
<ModelExtraPrices model={selectedModel} />
</div>
</div>

20
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -30543,17 +30543,12 @@ export interface components {
worker_registry?: components["schemas"]["WorkerRegistryEntry"][] | null;
};
/** ConfigurableClientsideParamsCustomAuth */
"ConfigurableClientsideParamsCustomAuth-Input": {
ConfigurableClientsideParamsCustomAuth: {
/** Api Base */
api_base: string;
} & {
[key: string]: unknown;
};
/** ConfigurableClientsideParamsCustomAuth */
"ConfigurableClientsideParamsCustomAuth-Output": {
/** Api Base */
api_base: string;
};
/** ContainerAuto */
ContainerAuto: {
/** File Ids */
@ -35412,7 +35407,7 @@ export interface components {
/** Complexity Router Default Model */
complexity_router_default_model?: string | null;
/** Configurable Clientside Auth Params */
configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Input"])[] | null;
configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth"])[] | null;
/** Cost Per Second */
cost_per_second?: number | null;
/** Custom Llm Provider */
@ -38490,10 +38485,15 @@ export interface components {
*/
warn_within_days: number;
};
/** ModelGroupClientsideAuthParam */
ModelGroupClientsideAuthParam: {
/** Api Base */
api_base: string;
};
/** ModelGroupInfoProxy */
ModelGroupInfoProxy: {
/** Configurable Clientside Auth Params */
configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null;
configurable_clientside_auth_params?: (string | components["schemas"]["ModelGroupClientsideAuthParam"])[] | null;
/** Custom Metadata */
custom_metadata?: {
[key: string]: string | number | boolean;
@ -38584,6 +38584,8 @@ export interface components {
supports_web_search: boolean;
/** Tpm */
tpm?: number | null;
} & {
[key: string]: unknown;
};
/** ModelInfoDelete */
ModelInfoDelete: {
@ -50815,7 +50817,7 @@ export interface components {
/** Complexity Router Default Model */
complexity_router_default_model?: string | null;
/** Configurable Clientside Auth Params */
configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Input"])[] | null;
configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth"])[] | null;
/** Cost Per Second */
cost_per_second?: number | null;
/** Custom Llm Provider */