diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 88029615ba8..71a2577092b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -94,6 +94,7 @@ from litellm.types.utils import ( LlmProviders, LlmProvidersSet, ModelInfo, + ServiceTier, StandardBuiltInToolsParams, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -614,7 +615,9 @@ def cost_per_token( # noqa: PLR0915 service_tier=service_tier, ) elif custom_llm_provider == "anthropic": - return anthropic_cost_per_token(model=model, usage=usage_block) + return anthropic_cost_per_token( + model=model, usage=usage_block, service_tier=service_tier + ) elif custom_llm_provider == "bedrock": return bedrock_cost_per_token( model=model, usage=usage_block, service_tier=service_tier @@ -1224,6 +1227,16 @@ def completion_cost( # noqa: PLR0915 if service_tier is None and optional_params is not None: service_tier = optional_params.get("service_tier") + # A request-level service_tier only prices the request when it is a + # concrete billable tier string. "auto" is a routing preference and any + # non-string value is not a billable tier, so defer to the tier the + # provider reports on the response/usage instead of crashing or mispricing + if ( + not isinstance(service_tier, str) + or service_tier.lower() == ServiceTier.AUTO.value + ): + service_tier = None + # Extract service_tier from completion_response if not provided if service_tier is None and completion_response is not None: if isinstance(completion_response, BaseModel): diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 7fb7be7ab84..6feaf2734e9 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -17,7 +17,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, ) from litellm.integrations.otel.plumbing.providers import to_otel_span_kind -from litellm.integrations.otel.model.semconv import Error +from litellm.integrations.otel.model.semconv import Error, ExceptionEvent from litellm.integrations.otel.model.spans import ( SPAN_REGISTRY, SpanRole, @@ -179,9 +179,17 @@ class SpanEmitter: else None ) if error and (error.error_type or error.message): - span.set_attribute(Error.TYPE, error.error_type or "error") - span.set_status( - Status(StatusCode.ERROR, error.message or error.error_type or "error") + error_type = error.error_type or "error" + message = error.message or error.error_type or "error" + span.set_attribute(Error.TYPE, error_type) + span.set_status(Status(StatusCode.ERROR, message)) + # Carry the full message on the standard ``exception`` event so backends + # map it as full text under ``exception.message``. Setting it as a bare + # string attribute instead lets backends like Elasticsearch dynamic-map + # it to a ``keyword`` capped at 1024 chars, truncating the message. + span.add_event( + ExceptionEvent.NAME, + {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, ) # On success leave the status UNSET (the semconv default) rather than # forcing OK — that matches the FastAPI server span and avoids implying a diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 7df07f30a01..bb93a357516 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -146,6 +146,21 @@ class Error: TYPE: Final = "error.type" +class ExceptionEvent: + """OTel exception-event name and attribute keys (semconv ``exception.*``). + + The full error message rides ``exception.message`` on a span event rather than + a custom string attribute. Backends recognise these semantic-convention names + and map them as full text; an unrecognised key (e.g. ``error_message``) falls + into the default dynamic template, which truncates strings to a 1024-char + ``keyword``. + """ + + NAME: Final = "exception" + TYPE: Final = "exception.type" + MESSAGE: Final = "exception.message" + + class Server: ADDRESS: Final = "server.address" PORT: Final = "server.port" diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index cc30db0ebad..7b87ed02a69 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2205,6 +2205,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): inference_geo: Optional[str] = None if "inference_geo" in _usage and _usage["inference_geo"] is not None: inference_geo = _usage["inference_geo"] + service_tier = cast( + str | None, + _usage.get("service_tier"), # any-ok: untyped usage dict + ) if ( "cache_creation_input_tokens" in _usage @@ -2298,6 +2302,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ), inference_geo=inference_geo, speed=speed, + service_tier=service_tier, ) return usage diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 6a031498dae..44081ea9e79 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -18,7 +18,9 @@ if TYPE_CHECKING: import litellm -def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: +def _compute_cache_only_cost( + model_info: "ModelInfo", usage: "Usage", service_tier: str | None = None +) -> float: """ Return only the cache-related portion of the prompt cost (cache read + cache write). @@ -36,7 +38,9 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost, - ) = _get_token_base_cost(model_info=model_info, usage=usage) + ) = _get_token_base_cost( + model_info=model_info, usage=usage, service_tier=service_tier + ) cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost @@ -56,19 +60,26 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: return cache_cost -def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: +def cost_per_token( + model: str, usage: "Usage", service_tier: str | None = None +) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information + - service_tier: the service tier the request was served at (e.g. "priority"), + read from the Anthropic response usage and used to select tier-specific pricing Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ prompt_cost, completion_cost = generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="anthropic" + model=model, + usage=usage, + custom_llm_provider="anthropic", + service_tier=service_tier, ) # Apply provider_specific_entry multipliers for geo/speed routing @@ -89,7 +100,9 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: multiplier *= provider_specific_entry.get("fast", 1.0) if multiplier != 1.0: - cache_cost = _compute_cache_only_cost(model_info=model_info, usage=usage) + cache_cost = _compute_cache_only_cost( + model_info=model_info, usage=usage, service_tier=service_tier + ) prompt_cost = (prompt_cost - cache_cost) * multiplier + cache_cost completion_cost *= multiplier except Exception: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c9fe3a0c88e..ad161457326 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -376,6 +376,8 @@ class LiteLLMRoutes(enum.Enum): # vector stores "/vector_stores", "/v1/vector_stores", + "/vector_stores/{vector_store_id}", + "/v1/vector_stores/{vector_store_id}", "/vector_stores/{vector_store_id}/search", "/v1/vector_stores/{vector_store_id}/search", "/vector_stores/{vector_store_id}/files", diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py new file mode 100644 index 00000000000..3a70377037d --- /dev/null +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -0,0 +1,167 @@ +"""Team-scoped (BYOK) model-name translation for the model listing endpoints. + +`/v1/models`, `/models`, and `GET /v1/models/{id}` should surface the public +`team_public_model_name` rather than the internal routing key +`model_name_{team_id}_{uuid}`, consistent with `/v1/model/info`. The internal +key still routes regardless; this is a presentation-layer swap only and does not +touch access-group or auth semantics (see issue #28382). Operators can pin the +legacy internal names with `general_settings.use_team_public_model_name: false`. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from litellm.router import Router + + +class TeamModelNameTranslator: + """Translates internal team routing keys to their public names for the model + listing/retrieve responses. Stateless; the live router and general_settings + are injected per call so the unit tests can drive it without globals. + """ + + @staticmethod + def _internal_public_pair(model: object) -> tuple[str, str] | None: + """`(internal_routing_key, public_name)` for a team-scoped row, else None.""" + if not isinstance(model, dict): + return None + model_dict = cast(dict[str, object], model) # any-ok: checked + model_info_raw: object = model_dict.get("model_info") + if not isinstance(model_info_raw, Mapping): + return None + model_info = cast(Mapping[str, object], model_info_raw) # any-ok: checked + team_id = model_info.get("team_id") + team_public = model_info.get("team_public_model_name") + name = model_dict.get("model_name") + if ( + isinstance(team_id, str) + and isinstance(team_public, str) + and isinstance(name, str) + and team_id + and team_public + and name.startswith(f"model_name_{team_id}_") + ): + return name, team_public + return None + + @staticmethod + def _is_enabled(general_settings: Mapping[str, object]) -> bool: + return general_settings.get("use_team_public_model_name", True) is not False + + @staticmethod + def build_internal_to_public_map( + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> dict[str, str]: + """Internal team routing key -> public `team_public_model_name`. + + Empty when disabled via the legacy flag, the router is absent, or the + router model list is malformed. + """ + if llm_router is None or not TeamModelNameTranslator._is_enabled( + general_settings + ): + return {} + router_model_list = llm_router.get_model_list() + if not isinstance(router_model_list, list): + return {} + return dict( + pair + for pair in ( + TeamModelNameTranslator._internal_public_pair(model) + for model in router_model_list + ) + if pair is not None + ) + + @staticmethod + def _response_to_lookup_map( + model_names: list[str], + internal_to_public: dict[str, str], + ) -> dict[str, str]: + """Map each public response id to the first internal lookup id seen in + `model_names`, preserving first-occurrence order. First-wins keeps list + and retrieve in agreement on which accessible deployment a shared public + id resolves to: a global iterated before a colliding team alias stays + the listed entry, and sibling team rows collapse to their first + occurrence. + """ + result: dict[str, str] = {} + for name in model_names: + result.setdefault(internal_to_public.get(name, name), name) + return result + + @staticmethod + def listing_entries( + model_names: list[str], + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> list[tuple[str, str]]: + """`(response_id, metadata_lookup_id)` for each listed model, de-duplicated + by response_id while preserving order. + + For team-scoped rows `response_id` is the public name shown to the client, + while `metadata_lookup_id` stays the internal routing key so downstream + metadata/fallback lookups (keyed by the routing name) still resolve. The + lookup id is always one of `model_names` (the caller's accessible set), so + a public name shared across teams never resolves to another team's + internal key. Both ids are identical for unmapped names (globals, + access-group keys). + """ + internal_to_public = TeamModelNameTranslator.build_internal_to_public_map( + llm_router, general_settings + ) + if not internal_to_public: + return [(name, name) for name in model_names] + return list( + TeamModelNameTranslator._response_to_lookup_map( + model_names, internal_to_public + ).items() + ) + + @staticmethod + def translate_listing( + model_names: list[str], + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> list[str]: + """Public-name view of `model_names` (the `response_id` of each listing + entry). Sibling deployments sharing a public name collapse to one entry + while preserving order; unmapped names pass through. + """ + return [ + entry[0] + for entry in TeamModelNameTranslator.listing_entries( + model_names, llm_router, general_settings + ) + ] + + @staticmethod + def resolve_public_name( + model_id: str, + available_models: list[str], + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> str: + """Resolve a public team name back to the internal routing key the router + indexes by, so `GET /v1/models/{id}` accepts the name the listing returns. + + Resolution is restricted to `available_models` (the caller's accessible + set) so colliding public names across teams never resolve across an access + boundary. Uses the same first-occurrence dedup as `listing_entries` so a + public id advertised by `/v1/models` resolves to the same internal + deployment that the listing's metadata was built from. Returns `model_id` + unchanged when it is not an accessible public team name (already-internal + names and globals pass through). + """ + internal_to_public = TeamModelNameTranslator.build_internal_to_public_map( + llm_router, general_settings + ) + if not internal_to_public: + return model_id + return TeamModelNameTranslator._response_to_lookup_map( + available_models, internal_to_public + ).get(model_id, model_id) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index cc0d06e4f40..694924f2b90 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -10,6 +10,8 @@ from litellm.types.utils import SpecialEnums if TYPE_CHECKING: from fastapi import Request + from litellm.router import Router + def _is_base64_encoded_unified_file_id(b64_uid: str) -> Union[str, Literal[False]]: # Ensure b64_uid is a string and not a mock object @@ -296,6 +298,92 @@ def get_credentials_for_model( return credentials +def get_team_provider_credentials( + llm_router: Optional["Router"], + team_models: List[str], + custom_llm_provider: str, + team_id: Optional[str] = None, +) -> Optional[dict]: + """ + Resolve upstream credentials for a provider-scoped file operation + (e.g. GET /v1/files), which doesn't pin a model. + + Priority: + 1. The team's own (BYOK) deployment for this provider — a deployment whose + ``model_info.team_id`` matches ``team_id``. This keeps team-scoped listings + on the team's own provider account/key instead of a shared global one. + 2. Fallback: any deployment the team is granted access to for this provider, + expanding wildcard routes and the all-proxy-models sentinel. + + Credential lookup is always scoped to the team's allowlist, so a team can + never resolve a provider key for a deployment it isn't authorized to use. + Returns None when the router is unavailable or no authorized deployment + matches, so the caller can fall back to default credential resolution. + """ + if llm_router is None: + return None + + def _provider_credentials(model_id: str) -> Optional[dict]: + credentials = llm_router.get_deployment_credentials_with_provider( + model_id=model_id + ) + if ( + credentials is not None + and credentials.get("custom_llm_provider") == custom_llm_provider + ): + return credentials + return None + + # 1. Prefer the team's own BYOK deployment, matched by model_info.team_id. + if team_id is not None: + for deployment in llm_router.model_list or []: + model_info = deployment.get("model_info") or {} + if model_info.get("team_id") != team_id: + continue + deployment_id = model_info.get("id") + if deployment_id is None: + continue + credentials = _provider_credentials(deployment_id) + if credentials is not None: + return credentials + + # 2. Fall back to deployments the team is allowed to access. The + # all-proxy-models sentinel isn't expanded by get_complete_model_list, so + # normalize it to an empty allowlist, which defers to the team-scoped + # proxy model list. A team with a restricted allowlist (e.g. anthropic + # only) therefore never resolves another provider's key. + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.model_checks import get_complete_model_list + + grants_all_models = SpecialModelNames.all_proxy_models.value in team_models + effective_team_models = [] if grants_all_models else team_models + + proxy_model_list = llm_router.get_model_names(team_id=team_id) + model_access_groups = llm_router.get_model_access_groups() + models_to_try = list( + dict.fromkeys( + get_complete_model_list( + key_models=[], + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + ) + ) + ) + for model_name in models_to_try: + credentials = _provider_credentials(model_name) + if credentials is not None: + return credentials + + return None + + def prepare_data_with_credentials( data: dict, credentials: dict, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 378cbbda89c..57b6d111e0e 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -51,6 +51,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( encode_file_id_with_model, extract_file_creation_params, get_credentials_for_model, + get_team_provider_credentials, handle_model_based_routing, prepare_data_with_credentials, ) @@ -1344,14 +1345,20 @@ async def list_files( status_code=400, detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", ) - ## Use router to list fine-tuning jobs for that model if llm_router is None: raise HTTPException( status_code=500, detail="LLM Router not initialized. Ensure models added to proxy.", ) - data["model"] = target_model_names_list[0] - response = await llm_router.afile_list( + credentials = get_credentials_for_model( + llm_router=llm_router, + model_id=target_model_names_list[0], + operation_context="file list", + ) + prepare_data_with_credentials(data=data, credentials=credentials) + response = await litellm.afile_list( + custom_llm_provider=credentials["custom_llm_provider"], + purpose=purpose, **data, ) else: @@ -1363,6 +1370,18 @@ async def list_files( or "openai" ) + # No model/target_model_names pinned: resolve upstream credentials from + # the team's deployment for this provider so the call is authenticated + # against the team's own account (e.g. the team's openai deployment). + team_credentials = get_team_provider_credentials( + llm_router=llm_router, + team_models=user_api_key_dict.team_models or [], + custom_llm_provider=custom_llm_provider, + team_id=user_api_key_dict.team_id, + ) + if team_credentials is not None: + prepare_data_with_credentials(data=data, credentials=team_credentials) + response = await litellm.afile_list( custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a6413138bd5..3f7ec4d74fa 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15,6 +15,7 @@ import threading import time import traceback import warnings +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import ( TYPE_CHECKING, @@ -302,6 +303,7 @@ from litellm.proxy.common_utils.load_config_utils import ( get_config_file_contents_from_gcs, get_file_contents_from_s3, ) +from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) @@ -8202,6 +8204,7 @@ async def model_list( include_metadata: Optional[bool] = False, fallback_type: Optional[str] = None, scope: Optional[str] = None, + healthy_only: Optional[bool] = False, ): """ Use `/model/info` - to get detailed model information, example - pricing, mode, etc. @@ -8215,9 +8218,20 @@ async def model_list( - scope: Optional scope parameter. Currently only accepts "expand". When scope=expand is passed, proxy admins, team admins, and org admins will receive all proxy models as if they are a proxy admin. + - healthy_only: When true, hide models whose backing deployments are all marked + unhealthy by background health checks. Requires + `background_health_checks: true` in general_settings; without + health state the listing is returned unfiltered (fail open). + Models expanded from wildcard routes (e.g. `openai/*`) are not + filtered, and nothing is hidden when `allowed_fails_policy` is + configured (cooldown remains the sole exclusion mechanism). + Hiding is presentation-only: a hidden model can still be + called directly. """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj + settings = cast(dict[str, object], general_settings) # any-ok: legacy settings + from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, ) @@ -8248,6 +8262,19 @@ async def model_list( llm_router.get_fully_blocked_model_names() if llm_router is not None else set() ) + # Opt-in: also hide models whose deployments are all unhealthy per background + # health checks. Empty when health state is unavailable or stale (fail open). + unhealthy_names: Set[str] = set() + if healthy_only and llm_router is not None: + unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names() + if not unhealthy_names: + verbose_proxy_logger.debug( + "healthy_only=true but no unhealthy deployment state is available " + "(requires background_health_checks); returning unfiltered model list" + ) + + hidden_names = blocked_names | unhealthy_names + # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: # Get all proxy models as if user is a proxy admin @@ -8280,20 +8307,25 @@ async def model_list( only_model_access_groups=only_model_access_groups or False, ) - # Hide paused models from the public listing (admins manage them via /model/info) - if blocked_names: - all_models = [m for m in all_models if m not in blocked_names] + # Hide paused/unhealthy models from the public listing + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] - # Build response data with all proxy models + # Surface the public team name by default; legacy internal keys via flag. + # The internal routing key drives the metadata/fallback lookup, while the + # public name is what the client sees as the model id. model_data = [] - for model in all_models: + for response_id, lookup_id in TeamModelNameTranslator.listing_entries( + all_models, llm_router, settings + ): model_info = create_model_info_response( - model_id=model, + model_id=lookup_id, provider="openai", include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, ) + model_info["id"] = response_id model_data.append(model_info) return dict( @@ -8317,20 +8349,25 @@ async def model_list( user_api_key_cache=user_api_key_cache, ) - # Hide paused models from the public listing (admins manage them via /model/info) - if blocked_names: - all_models = [m for m in all_models if m not in blocked_names] + # Hide paused/unhealthy models from the public listing + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] - # Build response data + # Surface the public team name by default; legacy internal keys via flag. + # The internal routing key drives the metadata/fallback lookup, while the + # public name is what the client sees as the model id. model_data = [] - for model in all_models: + for response_id, lookup_id in TeamModelNameTranslator.listing_entries( + all_models, llm_router, settings + ): model_info = create_model_info_response( - model_id=model, + model_id=lookup_id, provider="openai", include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, ) + model_info["id"] = response_id model_data.append(model_info) return dict( @@ -8352,6 +8389,8 @@ async def model_list( async def model_info( model_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + team_id: Optional[str] = None, + healthy_only: Optional[bool] = False, ): """ Retrieve information about a specific model accessible to your API key. @@ -8361,16 +8400,21 @@ async def model_info( Follows OpenAI API specification for individual model retrieval. https://platform.openai.com/docs/api-reference/models/retrieve + + Query parameters mirror `/v1/models` so the same caller context (team + scoping, health filtering, paused deployments) drives both endpoints; the + listing's public id must resolve to the same internal deployment here. """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj + settings = cast(dict[str, object], general_settings) # any-ok: legacy settings + from litellm.proxy.utils import ( create_model_info_response, get_available_models_for_user, validate_model_access, ) - # Get available models for the user all_models = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, llm_router=llm_router, @@ -8378,21 +8422,43 @@ async def model_info( user_model=user_model, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, - team_id=None, + team_id=team_id, include_model_access_groups=False, only_model_access_groups=False, return_wildcard_routes=False, user_api_key_cache=user_api_key_cache, ) + # Mirror /v1/models' visibility filter so first-occurrence resolution + # cannot land on a deployment the listing had hidden. + blocked_names = ( + llm_router.get_fully_blocked_model_names() if llm_router is not None else set() + ) + unhealthy_names: set[str] = set() + if healthy_only and llm_router is not None: + unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names() + hidden_names = blocked_names | unhealthy_names + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] + + internal_to_public = TeamModelNameTranslator.build_internal_to_public_map( + llm_router, settings + ) + resolved_model_id = TeamModelNameTranslator.resolve_public_name( + model_id=model_id, + available_models=all_models, + llm_router=llm_router, + general_settings=settings, + ) + # Validate that the requested model is accessible - validate_model_access(model_id=model_id, available_models=all_models) + validate_model_access(model_id=resolved_model_id, available_models=all_models) # Get provider information from the router deployment if llm_router is None: raise HTTPException(status_code=500, detail="Router not initialized") - deployment = llm_router.get_deployment_by_model_group_name(model_id) + deployment = llm_router.get_deployment_by_model_group_name(resolved_model_id) if deployment is None: raise HTTPException( status_code=404, @@ -8402,9 +8468,9 @@ async def model_info( # Use the actual litellm model from the deployment to get provider info _, provider, _, _ = litellm.get_llm_provider(model=deployment.litellm_params.model) - # Return the model information in the same format as the list endpoint + response_id = internal_to_public.get(resolved_model_id, model_id) return create_model_info_response( - model_id=model_id, + model_id=response_id, provider=provider, include_metadata=False, fallback_type=None, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c15c37f6ad8..d86cd38a51c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -45,6 +45,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.proxy.model_listing import ModelInfoResponse from litellm.types.utils import CallTypes, CallTypesLiteral try: @@ -6231,56 +6232,39 @@ def create_model_info_response( include_metadata: bool = False, fallback_type: Optional[str] = None, llm_router: Optional["Router"] = None, -) -> dict: +) -> ModelInfoResponse: """ - Create a standardized model info response. + Create a standardized OpenAI-compatible model object. - Args: - model_id: The model ID - provider: The model provider - include_metadata: Whether to include metadata - fallback_type: Type of fallbacks to include - llm_router: LiteLLM router instance - - Returns: - Dictionary containing model information + When include_metadata is true, attaches the model's configured fallbacks + (resolved via the router under fallback_type, defaulting to "general"). + Raises HTTPException(400) for an unknown fallback_type. """ from litellm.proxy.auth.model_checks import get_all_fallbacks - model_info = { + base: ModelInfoResponse = { "id": model_id, "object": "model", "created": DEFAULT_MODEL_CREATED_AT_TIME, "owned_by": provider, } + if not include_metadata: + return base - # Add metadata if requested - if include_metadata: - metadata = {} - - # Default fallback_type to "general" if include_metadata is true - effective_fallback_type = ( - fallback_type if fallback_type is not None else "general" + effective_fallback_type = fallback_type if fallback_type is not None else "general" + valid_fallback_types = ("general", "context_window", "content_policy") + if effective_fallback_type not in valid_fallback_types: + raise HTTPException( + status_code=400, + detail=f"Invalid fallback_type. Must be one of: {list(valid_fallback_types)}", ) - # Validate fallback_type - valid_fallback_types = ["general", "context_window", "content_policy"] - if effective_fallback_type not in valid_fallback_types: - raise HTTPException( - status_code=400, - detail=f"Invalid fallback_type. Must be one of: {valid_fallback_types}", - ) - - fallbacks = get_all_fallbacks( - model=model_id, - llm_router=llm_router, - fallback_type=effective_fallback_type, - ) - metadata["fallbacks"] = fallbacks - - model_info["metadata"] = metadata - - return model_info + fallbacks = get_all_fallbacks( + model=model_id, + llm_router=llm_router, + fallback_type=effective_fallback_type, + ) + return {**base, "metadata": {"fallbacks": fallbacks}} def validate_model_access( diff --git a/litellm/router.py b/litellm/router.py index d0f4e5ff44d..15de3d51175 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9948,6 +9948,63 @@ class Router: name for name, fully_blocked in blocked_by_name.items() if fully_blocked } + async def async_get_fully_unhealthy_model_names(self) -> Set[str]: + """ + Returns the set of model names where every backing deployment is currently + marked unhealthy by background health checks (and the health state is not stale). + + Used by `/v1/models?healthy_only=true` to hide models that cannot serve any + request. A model with at least one healthy (or unknown-health) deployment + remains visible. Returns an empty set when no health state is available, so + callers fail open to the unfiltered listing. + + Notes: + - Mirrors `_async_filter_health_check_unhealthy_deployments`: when + `allowed_fails_policy` is set, cooldown is the sole routing exclusion + mechanism, so nothing is hidden here either. + - Team-specific public model names (`team_public_model_name`) are + aggregated alongside `model_name`, so team aliases of fully-unhealthy + deployments are hidden too (unlike `get_fully_blocked_model_names`, + which matches `model_name` only). + - Wildcard routes (e.g. `openai/*`) are matched by their literal + deployment name only; models expanded from a wildcard route are not + hidden (fail open). + - Intentionally diverges from the routing-time safety net (which + bypasses the health filter when every candidate is unhealthy and + still attempts the request): hiding here is presentation-only — + it answers "should this model be advertised?", not "should a + request for it still be attempted?". A hidden model can still be + called directly. + """ + if self.allowed_fails_policy is not None: + return set() + unhealthy_ids = ( + await self.health_state_cache.async_get_unhealthy_deployment_ids() + ) + if not unhealthy_ids: + return set() + deployments = self.get_model_list() or [] + unhealthy_by_name: Dict[str, bool] = {} + for deployment in deployments: + model_info = deployment.get("model_info") or {} + names = [deployment.get("model_name") or ""] + team_public_model_name = model_info.get("team_public_model_name") + if team_public_model_name: + names.append(team_public_model_name) + is_unhealthy = model_info.get("id") in unhealthy_ids + for name in names: + if not name: + continue + if name in unhealthy_by_name: + unhealthy_by_name[name] = unhealthy_by_name[name] and is_unhealthy + else: + unhealthy_by_name[name] = is_unhealthy + return { + name + for name, fully_unhealthy in unhealthy_by_name.items() + if fully_unhealthy + } + def _get_team_specific_model( self, deployment: DeploymentTypedDict, team_id: Optional[str] = None ) -> Optional[str]: diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py new file mode 100644 index 00000000000..c3330da0d66 --- /dev/null +++ b/litellm/types/proxy/model_listing.py @@ -0,0 +1,21 @@ +"""Response types for the model listing/retrieve endpoints (/v1/models, /models).""" + +from typing import Literal + +from typing_extensions import NotRequired, TypedDict + + +class ModelInfoMetadata(TypedDict): + fallbacks: list[str] + + +class ModelInfoResponse(TypedDict): + """OpenAI-compatible model object. `metadata` is present only when the + endpoint is called with include_metadata=true. + """ + + id: str + object: Literal["model"] + created: int + owned_by: str + metadata: NotRequired[ModelInfoMetadata] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0200d1a0831..220d1861eb8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3635,6 +3635,7 @@ class SpecialEnums(Enum): class ServiceTier(Enum): """Enum for service tier types used in cost calculations.""" + AUTO = "auto" FLEX = "flex" PRIORITY = "priority" diff --git a/pyproject.toml b/pyproject.toml index 4a456b2c9a7..b5ef93f033c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.89.1" +version = "1.89.2" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.14" @@ -264,7 +264,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.89.1" +version = "1.89.2" version_files = [ "pyproject.toml:^version", ] diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index fef1d23d867..a184798b503 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -906,7 +906,10 @@ class BaseLLMChatTest(ABC): { "type": "image_url", "image_url": { - "url": "https://www.gstatic.com/webp/gallery/1.webp", + # sha-pinned in-repo logo via jsdelivr; gstatic's + # robots.txt blocks server-side fetchers (e.g. + # Anthropic), which 400s the request. + "url": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0/ui/litellm-dashboard/public/assets/logos/litellm_logo.jpg", "detail": detail, }, }, diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 86d84bd8100..c3fa3f424b5 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -410,6 +410,90 @@ def test_emitter_without_call_id_is_not_deduped(): assert len(exporter.get_finished_spans()) == 2 +def _emit_error_span(message, error_type="litellm.APIError"): + from litellm.integrations.otel.emitter import SpanEmitter + + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg) + data = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=SpanError(error_type=error_type, message=message), + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + engine.emit(SpanRole.LLM_CALL, data) + (span,) = exporter.get_finished_spans() + return span + + +def _exception_event(span): + from litellm.integrations.otel.model.semconv import ExceptionEvent + + events = [e for e in span.events if e.name == ExceptionEvent.NAME] + assert len(events) == 1, "expected exactly one exception event" + return events[0] + + +def test_error_message_recorded_as_full_exception_event_untruncated(): + """Regression for the Elasticsearch keyword/ignore_above:1024 truncation. + + A long error message must survive intact on the standard ``exception`` + event under ``exception.message`` — not get dropped onto a bare string + attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK + must not truncate it either, so a 5000-char message stays 5000 chars. + """ + from litellm.integrations.otel.model.semconv import Error, ExceptionEvent + + long_message = "boom: " + "x" * 5000 + span = _emit_error_span(long_message, error_type="litellm.APIError") + + event = _exception_event(span) + assert event.attributes[ExceptionEvent.MESSAGE] == long_message + assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024 + assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError" + + # error.type stays a low-cardinality attribute; the message does NOT become a + # bare string attribute (which is what got truncated). + assert span.attributes[Error.TYPE] == "litellm.APIError" + assert ExceptionEvent.MESSAGE not in span.attributes + assert span.status.description == long_message + + +def test_success_span_records_no_exception_event(): + from litellm.integrations.otel.emitter import SpanEmitter + from litellm.integrations.otel.model.semconv import ExceptionEvent + + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg) + data = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model="gpt-4o", + response_id="resp-1", + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=("stop",), + error=None, + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + engine.emit(SpanRole.LLM_CALL, data) + (span,) = exporter.get_finished_spans() + assert all(e.name != ExceptionEvent.NAME for e in span.events) + + # --- service taxonomy: which calls become spans, and of what kind ----------- # diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index c91c9c3fdf4..ce75ccdccce 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3702,6 +3702,39 @@ def test_fast_mode_with_inference_geo(): assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10 +def test_calculate_usage_captures_service_tier(): + """ + Anthropic returns the assigned service tier on the response usage object + (e.g. ``"priority"``). It must be surfaced on the Usage object so it is + visible in logs and used to select tier-specific pricing. + """ + config = AnthropicConfig() + + usage_object = { + "input_tokens": 410, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 585, + "service_tier": "priority", + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + assert usage.service_tier == "priority" + + +def test_calculate_usage_service_tier_defaults_to_none(): + """A response without a service tier must not invent one.""" + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={"input_tokens": 10, "output_tokens": 5}, + reasoning_content=None, + ) + + assert usage.service_tier is None + + def test_fast_mode_parameter_in_supported_params(): """ Test that 'speed' is in the list of supported OpenAI params. diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 63b61954cf6..07b04961205 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1400,6 +1400,65 @@ def test_rag_routes_accessible_to_internal_user_viewer(): ) +@pytest.mark.parametrize( + "route", + [ + "/vector_stores/vs_123", + "/v1/vector_stores/vs_123", + "/vector_stores/vs_123/search", + "/v1/vector_stores/vs_123/search", + "/vector_stores/vs_123/files", + "/v1/vector_stores/vs_123/files", + ], +) +def test_vector_store_routes_are_llm_api_routes(route): + """Retrieve/update/delete on a single vector store must classify as LLM API routes. + + Regression for the missing bare `/v1/vector_stores/{vector_store_id}` entry in + `openai_routes` that left retrieve/update/delete blocked for internal roles + while `/search` and `/files` sub-routes worked. + """ + + assert RouteChecks.is_llm_api_route(route) is True + + +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +@pytest.mark.parametrize( + "method, route", + [ + ("GET", "/v1/vector_stores/vs_123"), + ("POST", "/v1/vector_stores/vs_123"), + ("DELETE", "/v1/vector_stores/vs_123"), + ], +) +def test_vector_store_crud_accessible_to_internal_roles(user_role, method, route): + """Internal user and internal viewer must reach vector store retrieve/update/delete. + + Object-level access is still gated by `assert_user_can_access_vector_store`; + this only verifies the route gate no longer 403s these roles. + """ + + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role) + request = MagicMock(spec=Request) + request.method = method + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role), + _user_role=user_role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + def test_videos_route_accessible_to_internal_users(): """ Test that internal users can access the videos routes. diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 5fc36b71f2b..007c04e0aff 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -1873,3 +1873,329 @@ def test_get_file_content_non_openai_provider_skips_streaming_handler( assert "stream" not in captured_kwargs mock_streaming_response.assert_not_awaited() proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_resolves_wildcard_deployment_credentials( + mocker: MockerFixture, monkeypatch +): + """ + GET /v1/files?target_model_names= must resolve the upstream api_key + from the matching (wildcard) deployment. Regression for the path routing + through llm_router.afile_list(model=...), which reached OpenAI without an + api_key and failed with "api_key client option must be set". + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + wildcard_router = Router( + model_list=[ + { + "model_name": "*", + "litellm_params": { + "model": "openai/*", + "api_key": "wildcard-openai-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.get( + "/v1/files?target_model_names=gpt-4o", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "wildcard-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_without_target_model_names_uses_team_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Plain GET /v1/files (no target_model_names) must resolve the upstream openai + api_key from the team's openai deployment instead of falling through to a + keyless OpenAI client. Regression for "api_key client option must be set". + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + wildcard_router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=["openai/*"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_restricted_team_does_not_leak_global_openai_credentials( + mocker: MockerFixture, monkeypatch +): + """ + A team whose allowlist only grants anthropic must NOT resolve a global + openai deployment's api_key for plain GET /v1/files. Regression for the + last-resort scan that ignored team access control. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "global-openai-key", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="anthropic-only-team", + team_models=["claude-opus-4-6"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") != "global-openai-key" + + +def test_list_files_prefers_team_byok_over_global_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + When a team has its own BYOK openai deployment (model_info.team_id set), plain + GET /v1/files must use the team's key, not a shared/global openai deployment. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "global-openai-key", + }, + }, + { + "model_name": "team-gpt-4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "team-byok-openai-key", + }, + "model_info": { + "id": "team-byok-deployment-id", + "team_id": "test-team", + "team_public_model_name": "team-gpt-4o", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=["team-gpt-4o"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-byok-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_with_all_proxy_models_team_uses_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Teams with all-proxy-models (or empty models) must still resolve openai + credentials for plain GET /v1/files. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, SpecialModelNames + + wildcard_router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=[SpecialModelNames.all_proxy_models.value], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index 6a8e0d15d8b..0f87fcda588 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -15,6 +15,7 @@ import pytest import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator from litellm.proxy.proxy_server import ( _get_proxy_model_info, _translate_model_name_for_response, @@ -593,3 +594,664 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey team_filter.assert_awaited_once() assert team_filter.await_args.kwargs["team_id"] == "other-team" assert team_filter.await_args.kwargs["all_models"] == [team_row] + + +@pytest.mark.asyncio +async def test_v1_models_translates_team_model_for_access_group_key(monkeypatch): + """Regression (#28382 sibling leak): a virtual key whose model access group + resolves to a team BYOK deployment must list the PUBLIC name in /v1/models, + not the internal routing key model_name_{team_id}_{uuid}. + + The /model/info read-path fix did not cover /v1/models, which builds from + bare model-name strings via access-group expansion. + """ + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + # Default behavior: listing surfaces public names. + monkeypatch.setattr(ps, "general_settings", {}) + + # virtual key granted access via the access group (no team membership) + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key) + + ids = [d["id"] for d in resp["data"]] + assert "tushar-gpt-4.1" in ids + assert "model_name_teamX_uuid9" not in ids + + +@pytest.mark.asyncio +async def test_v1_models_keeps_internal_names_when_public_name_flag_disabled( + monkeypatch, +): + """Compatibility override: /v1/models can still list the internal routing + name for consumers that scripted against those ids. Translation is enabled + by default and disabled via general_settings['use_team_public_model_name']. + """ + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {"use_team_public_model_name": False}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key) + + ids = [d["id"] for d in resp["data"]] + assert "model_name_teamX_uuid9" in ids # internal id preserved (backward-compat) + assert "tushar-gpt-4.1" not in ids + + +@pytest.mark.asyncio +async def test_v1_models_translates_team_model_with_metadata(monkeypatch): + """include_metadata=true must build metadata for the public model id.""" + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key, include_metadata=True) + + assert resp["data"] == [ + { + "id": "tushar-gpt-4.1", + "object": "model", + "created": 1677610602, + "owned_by": "openai", + "metadata": {"fallbacks": []}, + } + ] + + +@pytest.mark.asyncio +async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch): + """Regression: with include_metadata=true, fallbacks configured for a team + model under its internal routing key must still surface. The metadata lookup + has to run against the internal name, not the translated public name (which + the router's fallback config never keys on) -- otherwise fallbacks silently + drop to [].""" + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + # Fallbacks are keyed on the internal routing name, as the router stores them. + router.fallbacks = [{"model_name_teamX_uuid9": ["gpt-4o-backup"]}] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key, include_metadata=True) + + assert resp["data"] == [ + { + "id": "tushar-gpt-4.1", + "object": "model", + "created": 1677610602, + "owned_by": "openai", + "metadata": {"fallbacks": ["gpt-4o-backup"]}, + } + ] + + +@pytest.mark.asyncio +async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch): + """Regression: two teams can publish the same team_public_model_name. With + include_metadata=true a caller scoped to teamX must see teamX's fallbacks for + the shared public name, never teamY's. The metadata lookup has to stay within + the caller's accessible models; resolving the public name through a router-wide + reverse map could point it at another team's internal routing key.""" + team_x = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "idX", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + team_y = { + "model_name": "model_name_teamY_uuidZ", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "idY", + "team_id": "teamY", + "team_public_model_name": "tushar-gpt-4.1", # same public name, other team + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_x, team_y] + router.get_model_list.return_value = [team_x, team_y] + router.fallbacks = [ + {"model_name_teamX_uuid9": ["teamX-backup"]}, + {"model_name_teamY_uuidZ": ["teamY-backup"]}, + ] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key, include_metadata=True) + + assert resp["data"] == [ + { + "id": "tushar-gpt-4.1", + "object": "model", + "created": 1677610602, + "owned_by": "openai", + "metadata": {"fallbacks": ["teamX-backup"]}, + } + ] + + +def test_translate_team_model_names_for_listing_swaps_and_dedupes(): + """Internal team routing keys -> public name; sibling deployments sharing a + public name collapse to one entry (order preserved); globals untouched.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + { + "model_name": "model_name_teamX_uuidB", # sibling: same public name + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + {"model_name": "gpt-4o", "model_info": {"db_model": False}}, + ] + + out = TeamModelNameTranslator.translate_listing( + ["model_name_teamX_uuidA", "model_name_teamX_uuidB", "gpt-4o"], + router, + {}, + ) + assert out == ["tushar-gpt-4.1", "gpt-4o"] + + +def test_listing_entries_keep_internal_lookup_id_for_team_rows(): + """`listing_entries` returns (public response id, internal lookup id) so the + response shows the public name while metadata lookups keep the routing key. + Sibling deployments collapse to one entry; globals map to themselves.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + { + "model_name": "model_name_teamX_uuidB", # sibling: same public name + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + {"model_name": "gpt-4o", "model_info": {"db_model": False}}, + ] + + entries = TeamModelNameTranslator.listing_entries( + ["model_name_teamX_uuidA", "model_name_teamX_uuidB", "gpt-4o"], + router, + {}, + ) + # public id for the client; an internal routing key for the metadata lookup + assert entries[0][0] == "tushar-gpt-4.1" + assert entries[0][1].startswith("model_name_teamX_uuid") + assert entries[1] == ("gpt-4o", "gpt-4o") + assert len(entries) == 2 + + +def test_listing_entries_lookup_id_never_crosses_team_boundary(): + """Regression: when two teams share a team_public_model_name, the lookup id for + the shared public name must stay within the caller's accessible model_names and + never resolve to the other team's internal routing key (which would leak that + team's fallback metadata under include_metadata=true).""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "shared-name", + }, + }, + { + "model_name": "model_name_teamY_uuidB", # different team, same public name + "model_info": { + "team_id": "teamY", + "team_public_model_name": "shared-name", + }, + }, + ] + + # caller can only access teamX's internal key + entries = TeamModelNameTranslator.listing_entries( + ["model_name_teamX_uuidA"], router, {} + ) + + assert entries == [("shared-name", "model_name_teamX_uuidA")] + + +def test_listing_entries_global_wins_when_team_alias_collides_with_global(): + """Regression: when an accessible global model shares its name with a team + deployment's `team_public_model_name`, the listing must keep the global + entry rather than overwriting its lookup id with the colliding team's + internal routing key (which would surface the team's metadata under the + global id).""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "gpt-4o", + }, + }, + {"model_name": "gpt-4o", "model_info": {"db_model": False}}, + ] + + entries = TeamModelNameTranslator.listing_entries( + ["gpt-4o", "model_name_teamX_uuidA"], router, {} + ) + + assert entries == [("gpt-4o", "gpt-4o")] + + +def test_listing_and_resolve_agree_on_sibling_internal_key(): + """Regression: when two team deployments share a public name, listing and + retrieve must pick the same internal routing key, otherwise `/v1/models/{id}` + describes a different deployment than what the listing's metadata was built + from.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + { + "model_name": "model_name_teamX_uuidB", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + ] + available = ["model_name_teamX_uuidA", "model_name_teamX_uuidB"] + + [(_, listing_lookup)] = TeamModelNameTranslator.listing_entries( + available, router, {} + ) + resolve_lookup = TeamModelNameTranslator.resolve_public_name( + model_id="tushar-gpt-4.1", + available_models=available, + llm_router=router, + general_settings={}, + ) + + assert listing_lookup == resolve_lookup + + +def test_listing_entries_skips_empty_team_public_model_name(): + """Regression: a misconfigured row with `team_public_model_name: ""` must not + produce a listing entry with an empty `id`; the internal routing key should + pass through unchanged, matching `/v1/model/info`'s falsy-check behavior.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "", + }, + }, + ] + + entries = TeamModelNameTranslator.listing_entries( + ["model_name_teamX_uuidA"], router, {} + ) + + assert entries == [("model_name_teamX_uuidA", "model_name_teamX_uuidA")] + + +def test_listing_entries_passthrough_when_disabled(): + """Legacy flag / no router -> response id equals lookup id (no translation).""" + assert TeamModelNameTranslator.listing_entries(["a", "b"], None, {}) == [ + ("a", "a"), + ("b", "b"), + ] + + +def test_translate_team_model_names_for_listing_leaves_unmapped_names(): + """Names with no team mapping (globals, access-group keys) pass through.""" + router = MagicMock() + router.get_model_list.return_value = [ + {"model_name": "gpt-4o", "model_info": {"db_model": False}} + ] + + assert TeamModelNameTranslator.translate_listing( + ["gpt-4o", "beta-group"], router, {} + ) == ["gpt-4o", "beta-group"] + + +def test_translate_team_model_names_for_listing_none_router(): + """No router -> return the input list unchanged.""" + assert TeamModelNameTranslator.translate_listing(["a", "b"], None, {}) == ["a", "b"] + + +def test_translate_team_model_names_for_listing_respects_legacy_flag(): + """Operators can keep returning the legacy internal routing key.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + } + ] + + assert TeamModelNameTranslator.translate_listing( + ["model_name_teamX_uuidA"], router, {"use_team_public_model_name": False} + ) == ["model_name_teamX_uuidA"] + + +def _public_named_router(*team_rows: dict) -> MagicMock: + router = MagicMock() + router.get_model_list.return_value = list(team_rows) + return router + + +def test_resolve_public_name_to_internal_routing_key(): + """A public team name resolves back to the internal routing key the router + indexes by, so `GET /v1/models/{public_name}` can find the deployment.""" + router = _public_named_router(_team_row()) + + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={}, + ) + == "model_name_team-abc-123_4a6b8" + ) + + +def test_resolve_public_name_is_access_scoped_across_teams(): + """Two teams can publish the SAME public name. A caller's query must resolve + to the internal key they can actually access, never another team's.""" + # both rows share public name "team-claude-sonnet" + router = _public_named_router(_team_row(), _other_team_row()) + + # caller only has access to their own team's internal key + resolved = TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={}, + ) + assert resolved == "model_name_team-abc-123_4a6b8" + assert resolved != "model_name_team-other_9f2c1" + + +def test_resolve_public_name_unmapped_passes_through(): + """A public name with no accessible internal mapping is returned unchanged so + the caller hits the normal 404/access path; internal names pass through too.""" + router = _public_named_router(_team_row()) + + # not accessible -> unchanged (downstream validate_model_access will 404) + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=[], + llm_router=router, + general_settings={}, + ) + == "team-claude-sonnet" + ) + # already an internal routing key -> unchanged + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="model_name_team-abc-123_4a6b8", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={}, + ) + == "model_name_team-abc-123_4a6b8" + ) + + +def test_resolve_public_name_respects_legacy_flag(): + """With the legacy flag set, no public-name resolution happens.""" + router = _public_named_router(_team_row()) + + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={"use_team_public_model_name": False}, + ) + == "team-claude-sonnet" + ) + + +@pytest.mark.asyncio +async def test_retrieve_model_by_public_name_returns_200(monkeypatch): + """Regression: `GET /v1/models/{public_name}` must NOT 404. The listing + advertises the public team name, so retrieve must accept the same name, + resolve it to the internal routing key for lookup, and echo the public name + back as the model id.""" + import litellm + import litellm.proxy.utils as proxy_utils + + team_row = _team_row() + router = _public_named_router(team_row) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]), + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + resp = await ps.model_info(model_id="team-claude-sonnet", user_api_key_dict=key) + + assert resp["id"] == "team-claude-sonnet" + # lookup happened by the internal routing key, not the public name + router.get_deployment_by_model_group_name.assert_called_once_with( + "model_name_team-abc-123_4a6b8" + ) + + +@pytest.mark.asyncio +async def test_retrieve_model_by_internal_name_returns_public_id(monkeypatch): + """Regression: retrieving by the internal routing key must echo the SAME + public id `/v1/models` advertises for that deployment, not the path. Otherwise + a client iterating the listing's id and then retrieving each one would observe + a different id depending on which alias they queried by.""" + import litellm + import litellm.proxy.utils as proxy_utils + + router = _public_named_router(_team_row()) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]), + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + resp = await ps.model_info( + model_id="model_name_team-abc-123_4a6b8", user_api_key_dict=key + ) + + assert resp["id"] == "team-claude-sonnet" + + +@pytest.mark.asyncio +async def test_retrieve_model_by_internal_name_keeps_internal_id_when_flag_disabled( + monkeypatch, +): + """With `use_team_public_model_name=false`, retrieve must keep the internal + routing key as the response id, mirroring `/v1/models`' legacy output.""" + import litellm + import litellm.proxy.utils as proxy_utils + + router = _public_named_router(_team_row()) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {"use_team_public_model_name": False}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]), + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + resp = await ps.model_info( + model_id="model_name_team-abc-123_4a6b8", user_api_key_dict=key + ) + + assert resp["id"] == "model_name_team-abc-123_4a6b8" + + +@pytest.mark.asyncio +async def test_retrieve_model_by_inaccessible_public_name_404s(monkeypatch): + """A caller without access to a team model still gets 404 when retrieving by + its public name; resolution never crosses the access boundary.""" + import litellm + import litellm.proxy.utils as proxy_utils + + router = _public_named_router(_team_row()) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=[]), # caller has no access + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + with pytest.raises(ps.HTTPException) as exc_info: + await ps.model_info(model_id="team-claude-sonnet", user_api_key_dict=key) + + assert exc_info.value.status_code == 404 + router.get_deployment_by_model_group_name.assert_not_called() diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index aef91ed3c77..bc8b4474a7d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -359,6 +359,7 @@ ignored_keys = [ "metadata.additional_usage_values.cache_read_input_tokens", "metadata.additional_usage_values.inference_geo", "metadata.additional_usage_values.speed", + "metadata.additional_usage_values.service_tier", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", "metadata.user_api_key", diff --git a/tests/test_litellm/proxy/test_model_list_healthy_only.py b/tests/test_litellm/proxy/test_model_list_healthy_only.py new file mode 100644 index 00000000000..4ab33f3bf50 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_list_healthy_only.py @@ -0,0 +1,92 @@ +""" +Tests for the opt-in `healthy_only` filter on GET /v1/models (`model_list`). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.fixture +def patched_model_list(monkeypatch): + """Stub router + utility helpers used by `model_list`.""" + from litellm.proxy import utils as proxy_utils + + router = MagicMock() + router.get_fully_blocked_model_names = MagicMock(return_value=set()) + router.async_get_fully_unhealthy_model_names = AsyncMock( + return_value={"claude-sonnet"} + ) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "user_model", None) + + async def _fake_get_available_models_for_user(**kwargs): + return ["gpt-4", "claude-sonnet"] + + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + _fake_get_available_models_for_user, + ) + + def _fake_create_model_info_response(model_id, provider="openai", **kwargs): + return {"id": model_id, "object": "model", "created": 0, "owned_by": provider} + + monkeypatch.setattr( + proxy_utils, "create_model_info_response", _fake_create_model_info_response + ) + + return router + + +@pytest.mark.asyncio +async def test_model_list_healthy_only_hides_fully_unhealthy_models( + patched_model_list, +): + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + healthy_only=True, + ) + assert [m["id"] for m in response["data"]] == ["gpt-4"] + + +@pytest.mark.asyncio +async def test_model_list_default_keeps_unhealthy_models(patched_model_list): + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + assert [m["id"] for m in response["data"]] == ["gpt-4", "claude-sonnet"] + patched_model_list.async_get_fully_unhealthy_model_names.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_model_list_healthy_only_applies_to_scope_expand( + patched_model_list, monkeypatch +): + from litellm.proxy.auth import model_checks + from litellm.proxy.management_endpoints import common_utils + + async def _fake_admin(**kwargs): + return True + + monkeypatch.setattr(common_utils, "_user_has_admin_privileges", _fake_admin) + monkeypatch.setattr( + model_checks, + "get_complete_model_list", + lambda **kwargs: ["gpt-4", "claude-sonnet"], + ) + patched_model_list.get_model_names = MagicMock( + return_value=["gpt-4", "claude-sonnet"] + ) + patched_model_list.get_model_access_groups = MagicMock(return_value={}) + + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + scope="expand", + healthy_only=True, + ) + assert [m["id"] for m in response["data"]] == ["gpt-4"] diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 82a4a60bf82..ff4e36f593c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1948,6 +1948,221 @@ def test_completion_cost_service_tier_for_bedrock(): assert priority_cost > default_cost > flex_cost > 0 +def test_completion_cost_service_tier_for_anthropic(): + """ + Anthropic priority-tier requests must be priced at the priority rate. + + Regression for LIT-3771: the Anthropic cost route dropped ``service_tier``, + so priority requests (whose tier is reported on the response usage) were + always billed at the standard rate. The tier is captured by the + transformation and must flow through to ``generic_cost_per_token``. + """ + from litellm import completion_cost + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-service-tier-cost-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + } + } + ) + + def _cost_for_tier(service_tier): + usage = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 1000, + "output_tokens": 500, + "service_tier": service_tier, + }, + reasoning_content=None, + ) + response = ModelResponse(usage=usage, model=model) + return completion_cost( + completion_response=response, + model=model, + custom_llm_provider="anthropic", + ) + + standard_cost = _cost_for_tier("standard") + priority_cost = _cost_for_tier("priority") + + expected_standard = 1000 * 3e-6 + 500 * 15e-6 + assert standard_cost == pytest.approx(expected_standard) + # priority rates are exactly 2x standard for both input and output + assert priority_cost == pytest.approx(2 * standard_cost) + + +def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(): + """ + Proxy billing path regression for LIT-3771. + + Priority is opted into with ``service_tier="auto"``; Anthropic then serves + "priority" and reports it on the response usage. The proxy forwards the + request-level "auto" into ``completion_cost`` (via ``_response_cost_calculator``), + and that preference must not shadow the served tier, otherwise priority + requests are silently billed at the standard rate. + """ + from litellm import completion_cost + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-auto-tier-cost-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + } + } + ) + + usage = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 1000, + "output_tokens": 500, + "service_tier": "priority", + }, + reasoning_content=None, + ) + response = ModelResponse(usage=usage, model=model) + + cost = completion_cost( + completion_response=response, + model=model, + custom_llm_provider="anthropic", + service_tier="auto", + optional_params={"service_tier": "auto"}, + ) + + expected_priority = 1000 * 6e-6 + 500 * 30e-6 + assert cost == pytest.approx(expected_priority) + + +def test_completion_cost_non_string_service_tier_defers_to_served_tier(): + """ + Regression: a non-string request-level ``service_tier`` (reachable via + ``allowed_openai_params``/``drop_params``) must not crash cost tracking. + + Before the fix, ``completion_cost`` called ``service_tier.lower()`` on the + request-level value, so a dict raised ``AttributeError``. ``_response_cost_calculator`` + swallowed it and reported ``response_cost=None``, silently dropping the cost. + The non-string preference must be ignored so pricing defers to the tier the + provider actually served on the response usage. + """ + from litellm import completion_cost + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-non-string-tier-cost-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + } + } + ) + + usage = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 1000, + "output_tokens": 500, + "service_tier": "priority", + }, + reasoning_content=None, + ) + response = ModelResponse(usage=usage, model=model) + + cost = completion_cost( + completion_response=response, + model=model, + custom_llm_provider="anthropic", + optional_params={"service_tier": {"name": "auto"}}, + ) + + expected_priority = 1000 * 6e-6 + 500 * 30e-6 + assert cost == pytest.approx(expected_priority) + + +def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(): + """ + Regression for the cache/tier interaction in the Anthropic geo/speed path. + + When a request is served at "priority" and also carries a geo/speed + multiplier (here ``speed="fast"``), the cache portion is held out of the + multiplier so it is not scaled. That held-out cache cost must use the + served tier's cache rate; pricing it at the standard rate while the cache + embedded in ``prompt_cost`` is priced at the priority rate leaves a + ``(cache_priority - cache_standard)(multiplier - 1)`` billing error. + """ + from litellm.llms.anthropic.cost_calculation import ( + cost_per_token as anthropic_cost_per_token, + ) + from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-priority-cache-fast-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 0.3e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "cache_read_input_token_cost_priority": 0.6e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + "provider_specific_entry": {"fast": 2.0}, + } + } + ) + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200), + ) + usage.speed = "fast" + + prompt_cost, completion_cost = anthropic_cost_per_token( + model=model, usage=usage, service_tier="priority" + ) + + # non-cache input priced at the priority rate and scaled by the fast + # multiplier; the 200 cache-hit tokens priced at the priority cache rate + # and held out of the multiplier + expected_prompt = (1000 - 200) * 6e-6 * 2 + 200 * 0.6e-6 + expected_completion = 500 * 30e-6 * 2 + assert prompt_cost == pytest.approx(expected_prompt) + assert completion_cost == pytest.approx(expected_completion) + + def test_gemini_cache_tokens_details_no_negative_values(): """ Test for Issue #18750: Negative text_tokens with Gemini caching diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cd235d8de67..49ee871dac6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -4218,6 +4218,82 @@ def test_get_fully_blocked_model_names_treats_missing_key_as_unblocked(): assert router.get_fully_blocked_model_names() == set() +def _seed_unhealthy_states(router, unhealthy_ids, timestamp=None): + import time + + ts = timestamp if timestamp is not None else time.time() + router.health_state_cache.set_deployment_health_states( + { + uid: {"is_healthy": False, "timestamp": ts, "reason": "test_unhealthy"} + for uid in unhealthy_ids + } + ) + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_marks_name_when_all_unhealthy(): + router = _router_with_two_deployments([False, False]) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}) + assert await router.async_get_fully_unhealthy_model_names() == {"gpt-4o"} + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_keeps_name_when_partial(): + router = _router_with_two_deployments([False, False]) + _seed_unhealthy_states(router, {"dep-0"}) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_empty_without_health_state(): + router = _router_with_two_deployments([False, False]) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_ignores_stale_state(): + import time + + router = _router_with_two_deployments([False, False]) + stale_ts = time.time() - (router.health_state_cache.staleness_threshold + 10) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}, timestamp=stale_ts) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_includes_team_alias(): + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": { + "id": "dep-0", + "team_id": "team-1", + "team_public_model_name": "team-gpt", + }, + } + ] + ) + _seed_unhealthy_states(router, {"dep-0"}) + assert await router.async_get_fully_unhealthy_model_names() == { + "gpt-4o", + "team-gpt", + } + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_noop_with_allowed_fails_policy(): + from litellm.types.router import AllowedFailsPolicy + + router = _router_with_two_deployments([False, False]) + router.allowed_fails_policy = AllowedFailsPolicy(BadRequestErrorAllowedFails=1) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}) + assert await router.async_get_fully_unhealthy_model_names() == set() + + @pytest.mark.asyncio async def test_async_get_healthy_deployments_skips_blocked_deployment(): router = _router_with_two_deployments([True, False]) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c5560ca7f20..9699da9a6ef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6279,6 +6279,10 @@ export interface paths { * * Follows OpenAI API specification for individual model retrieval. * https://platform.openai.com/docs/api-reference/models/retrieve + * + * Query parameters mirror `/v1/models` so the same caller context (team + * scoping, health filtering, paused deployments) drives both endpoints; the + * listing's public id must resolve to the same internal deployment here. */ get: operations["model_info_models__model_id__get"]; put?: never; @@ -14472,6 +14476,10 @@ export interface paths { * * Follows OpenAI API specification for individual model retrieval. * https://platform.openai.com/docs/api-reference/models/retrieve + * + * Query parameters mirror `/v1/models` so the same caller context (team + * scoping, health filtering, paused deployments) drives both endpoints; the + * listing's public id must resolve to the same internal deployment here. */ get: operations["model_info_v1_models__model_id__get"]; put?: never; @@ -38145,7 +38153,10 @@ export interface operations { }; model_info_models__model_id__get: { parameters: { - query?: never; + query?: { + team_id?: string | null; + healthy_only?: boolean | null; + }; header?: never; path: { model_id: string; @@ -48211,7 +48222,10 @@ export interface operations { }; model_info_v1_models__model_id__get: { parameters: { - query?: never; + query?: { + team_id?: string | null; + healthy_only?: boolean | null; + }; header?: never; path: { model_id: string; diff --git a/uv.lock b/uv.lock index 81e099a7f06..efda9444fed 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-06-13T01:42:46.429412Z" +exclude-newer = "2026-06-15T02:02:32.823508Z" exclude-newer-span = "P3D" [manifest] @@ -3280,7 +3280,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.89.1" +version = "1.89.2" source = { editable = "." } dependencies = [ { name = "aiohttp" },