mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge pull request #30681 from BerriAI/litellm_backport_1_89_x_0617
chore(release): backport #30380, #30503, #30558, #30130, #30588, #30495, #30690 to stable/1.89.x and cut 1.89.2
This commit is contained in:
yuneng-jiang
GitHub
commit
94dae27b0c
27 changed files with 2099 additions and 75 deletions
|
|
@ -94,6 +94,7 @@ from litellm.types.utils import (
|
|||
LlmProviders,
|
||||
LlmProvidersSet,
|
||||
ModelInfo,
|
||||
ServiceTier,
|
||||
StandardBuiltInToolsParams,
|
||||
TranscriptionUsageDurationObject,
|
||||
TranscriptionUsageTokensObject,
|
||||
|
|
@ -614,7 +615,9 @@ def cost_per_token( # noqa: PLR0915
|
|||
service_tier=service_tier,
|
||||
)
|
||||
elif custom_llm_provider == "anthropic":
|
||||
return anthropic_cost_per_token(model=model, usage=usage_block)
|
||||
return anthropic_cost_per_token(
|
||||
model=model, usage=usage_block, service_tier=service_tier
|
||||
)
|
||||
elif custom_llm_provider == "bedrock":
|
||||
return bedrock_cost_per_token(
|
||||
model=model, usage=usage_block, service_tier=service_tier
|
||||
|
|
@ -1224,6 +1227,16 @@ def completion_cost( # noqa: PLR0915
|
|||
if service_tier is None and optional_params is not None:
|
||||
service_tier = optional_params.get("service_tier")
|
||||
|
||||
# A request-level service_tier only prices the request when it is a
|
||||
# concrete billable tier string. "auto" is a routing preference and any
|
||||
# non-string value is not a billable tier, so defer to the tier the
|
||||
# provider reports on the response/usage instead of crashing or mispricing
|
||||
if (
|
||||
not isinstance(service_tier, str)
|
||||
or service_tier.lower() == ServiceTier.AUTO.value
|
||||
):
|
||||
service_tier = None
|
||||
|
||||
# Extract service_tier from completion_response if not provided
|
||||
if service_tier is None and completion_response is not None:
|
||||
if isinstance(completion_response, BaseModel):
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from litellm.integrations.otel.model.payloads import (
|
|||
ServiceSpanData,
|
||||
)
|
||||
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
|
||||
from litellm.integrations.otel.model.semconv import Error
|
||||
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
|
||||
from litellm.integrations.otel.model.spans import (
|
||||
SPAN_REGISTRY,
|
||||
SpanRole,
|
||||
|
|
@ -179,9 +179,17 @@ class SpanEmitter:
|
|||
else None
|
||||
)
|
||||
if error and (error.error_type or error.message):
|
||||
span.set_attribute(Error.TYPE, error.error_type or "error")
|
||||
span.set_status(
|
||||
Status(StatusCode.ERROR, error.message or error.error_type or "error")
|
||||
error_type = error.error_type or "error"
|
||||
message = error.message or error.error_type or "error"
|
||||
span.set_attribute(Error.TYPE, error_type)
|
||||
span.set_status(Status(StatusCode.ERROR, message))
|
||||
# Carry the full message on the standard ``exception`` event so backends
|
||||
# map it as full text under ``exception.message``. Setting it as a bare
|
||||
# string attribute instead lets backends like Elasticsearch dynamic-map
|
||||
# it to a ``keyword`` capped at 1024 chars, truncating the message.
|
||||
span.add_event(
|
||||
ExceptionEvent.NAME,
|
||||
{ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message},
|
||||
)
|
||||
# On success leave the status UNSET (the semconv default) rather than
|
||||
# forcing OK — that matches the FastAPI server span and avoids implying a
|
||||
|
|
|
|||
|
|
@ -146,6 +146,21 @@ class Error:
|
|||
TYPE: Final = "error.type"
|
||||
|
||||
|
||||
class ExceptionEvent:
|
||||
"""OTel exception-event name and attribute keys (semconv ``exception.*``).
|
||||
|
||||
The full error message rides ``exception.message`` on a span event rather than
|
||||
a custom string attribute. Backends recognise these semantic-convention names
|
||||
and map them as full text; an unrecognised key (e.g. ``error_message``) falls
|
||||
into the default dynamic template, which truncates strings to a 1024-char
|
||||
``keyword``.
|
||||
"""
|
||||
|
||||
NAME: Final = "exception"
|
||||
TYPE: Final = "exception.type"
|
||||
MESSAGE: Final = "exception.message"
|
||||
|
||||
|
||||
class Server:
|
||||
ADDRESS: Final = "server.address"
|
||||
PORT: Final = "server.port"
|
||||
|
|
|
|||
|
|
@ -2205,6 +2205,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
inference_geo: Optional[str] = None
|
||||
if "inference_geo" in _usage and _usage["inference_geo"] is not None:
|
||||
inference_geo = _usage["inference_geo"]
|
||||
service_tier = cast(
|
||||
str | None,
|
||||
_usage.get("service_tier"), # any-ok: untyped usage dict
|
||||
)
|
||||
|
||||
if (
|
||||
"cache_creation_input_tokens" in _usage
|
||||
|
|
@ -2298,6 +2302,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
),
|
||||
inference_geo=inference_geo,
|
||||
speed=speed,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
return usage
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
|
||||
|
||||
def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float:
|
||||
def _compute_cache_only_cost(
|
||||
model_info: "ModelInfo", usage: "Usage", service_tier: str | None = None
|
||||
) -> float:
|
||||
"""
|
||||
Return only the cache-related portion of the prompt cost (cache read + cache write).
|
||||
|
||||
|
|
@ -36,7 +38,9 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float:
|
|||
cache_creation_cost,
|
||||
cache_creation_cost_above_1hr,
|
||||
cache_read_cost,
|
||||
) = _get_token_base_cost(model_info=model_info, usage=usage)
|
||||
) = _get_token_base_cost(
|
||||
model_info=model_info, usage=usage, service_tier=service_tier
|
||||
)
|
||||
|
||||
cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
|
||||
|
||||
|
|
@ -56,19 +60,26 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float:
|
|||
return cache_cost
|
||||
|
||||
|
||||
def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
|
||||
def cost_per_token(
|
||||
model: str, usage: "Usage", service_tier: str | None = None
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
|
||||
|
||||
Input:
|
||||
- model: str, the model name without provider prefix
|
||||
- usage: LiteLLM Usage block, containing anthropic caching information
|
||||
- service_tier: the service tier the request was served at (e.g. "priority"),
|
||||
read from the Anthropic response usage and used to select tier-specific pricing
|
||||
|
||||
Returns:
|
||||
Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
|
||||
"""
|
||||
prompt_cost, completion_cost = generic_cost_per_token(
|
||||
model=model, usage=usage, custom_llm_provider="anthropic"
|
||||
model=model,
|
||||
usage=usage,
|
||||
custom_llm_provider="anthropic",
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
# Apply provider_specific_entry multipliers for geo/speed routing
|
||||
|
|
@ -89,7 +100,9 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]:
|
|||
multiplier *= provider_specific_entry.get("fast", 1.0)
|
||||
|
||||
if multiplier != 1.0:
|
||||
cache_cost = _compute_cache_only_cost(model_info=model_info, usage=usage)
|
||||
cache_cost = _compute_cache_only_cost(
|
||||
model_info=model_info, usage=usage, service_tier=service_tier
|
||||
)
|
||||
prompt_cost = (prompt_cost - cache_cost) * multiplier + cache_cost
|
||||
completion_cost *= multiplier
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -376,6 +376,8 @@ class LiteLLMRoutes(enum.Enum):
|
|||
# vector stores
|
||||
"/vector_stores",
|
||||
"/v1/vector_stores",
|
||||
"/vector_stores/{vector_store_id}",
|
||||
"/v1/vector_stores/{vector_store_id}",
|
||||
"/vector_stores/{vector_store_id}/search",
|
||||
"/v1/vector_stores/{vector_store_id}/search",
|
||||
"/vector_stores/{vector_store_id}/files",
|
||||
|
|
|
|||
167
litellm/proxy/common_utils/model_listing_utils.py
Normal file
167
litellm/proxy/common_utils/model_listing_utils.py
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
"""Team-scoped (BYOK) model-name translation for the model listing endpoints.
|
||||
|
||||
`/v1/models`, `/models`, and `GET /v1/models/{id}` should surface the public
|
||||
`team_public_model_name` rather than the internal routing key
|
||||
`model_name_{team_id}_{uuid}`, consistent with `/v1/model/info`. The internal
|
||||
key still routes regardless; this is a presentation-layer swap only and does not
|
||||
touch access-group or auth semantics (see issue #28382). Operators can pin the
|
||||
legacy internal names with `general_settings.use_team_public_model_name: false`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
class TeamModelNameTranslator:
|
||||
"""Translates internal team routing keys to their public names for the model
|
||||
listing/retrieve responses. Stateless; the live router and general_settings
|
||||
are injected per call so the unit tests can drive it without globals.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _internal_public_pair(model: object) -> tuple[str, str] | None:
|
||||
"""`(internal_routing_key, public_name)` for a team-scoped row, else None."""
|
||||
if not isinstance(model, dict):
|
||||
return None
|
||||
model_dict = cast(dict[str, object], model) # any-ok: checked
|
||||
model_info_raw: object = model_dict.get("model_info")
|
||||
if not isinstance(model_info_raw, Mapping):
|
||||
return None
|
||||
model_info = cast(Mapping[str, object], model_info_raw) # any-ok: checked
|
||||
team_id = model_info.get("team_id")
|
||||
team_public = model_info.get("team_public_model_name")
|
||||
name = model_dict.get("model_name")
|
||||
if (
|
||||
isinstance(team_id, str)
|
||||
and isinstance(team_public, str)
|
||||
and isinstance(name, str)
|
||||
and team_id
|
||||
and team_public
|
||||
and name.startswith(f"model_name_{team_id}_")
|
||||
):
|
||||
return name, team_public
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_enabled(general_settings: Mapping[str, object]) -> bool:
|
||||
return general_settings.get("use_team_public_model_name", True) is not False
|
||||
|
||||
@staticmethod
|
||||
def build_internal_to_public_map(
|
||||
llm_router: "Router | None",
|
||||
general_settings: Mapping[str, object],
|
||||
) -> dict[str, str]:
|
||||
"""Internal team routing key -> public `team_public_model_name`.
|
||||
|
||||
Empty when disabled via the legacy flag, the router is absent, or the
|
||||
router model list is malformed.
|
||||
"""
|
||||
if llm_router is None or not TeamModelNameTranslator._is_enabled(
|
||||
general_settings
|
||||
):
|
||||
return {}
|
||||
router_model_list = llm_router.get_model_list()
|
||||
if not isinstance(router_model_list, list):
|
||||
return {}
|
||||
return dict(
|
||||
pair
|
||||
for pair in (
|
||||
TeamModelNameTranslator._internal_public_pair(model)
|
||||
for model in router_model_list
|
||||
)
|
||||
if pair is not None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _response_to_lookup_map(
|
||||
model_names: list[str],
|
||||
internal_to_public: dict[str, str],
|
||||
) -> dict[str, str]:
|
||||
"""Map each public response id to the first internal lookup id seen in
|
||||
`model_names`, preserving first-occurrence order. First-wins keeps list
|
||||
and retrieve in agreement on which accessible deployment a shared public
|
||||
id resolves to: a global iterated before a colliding team alias stays
|
||||
the listed entry, and sibling team rows collapse to their first
|
||||
occurrence.
|
||||
"""
|
||||
result: dict[str, str] = {}
|
||||
for name in model_names:
|
||||
result.setdefault(internal_to_public.get(name, name), name)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def listing_entries(
|
||||
model_names: list[str],
|
||||
llm_router: "Router | None",
|
||||
general_settings: Mapping[str, object],
|
||||
) -> list[tuple[str, str]]:
|
||||
"""`(response_id, metadata_lookup_id)` for each listed model, de-duplicated
|
||||
by response_id while preserving order.
|
||||
|
||||
For team-scoped rows `response_id` is the public name shown to the client,
|
||||
while `metadata_lookup_id` stays the internal routing key so downstream
|
||||
metadata/fallback lookups (keyed by the routing name) still resolve. The
|
||||
lookup id is always one of `model_names` (the caller's accessible set), so
|
||||
a public name shared across teams never resolves to another team's
|
||||
internal key. Both ids are identical for unmapped names (globals,
|
||||
access-group keys).
|
||||
"""
|
||||
internal_to_public = TeamModelNameTranslator.build_internal_to_public_map(
|
||||
llm_router, general_settings
|
||||
)
|
||||
if not internal_to_public:
|
||||
return [(name, name) for name in model_names]
|
||||
return list(
|
||||
TeamModelNameTranslator._response_to_lookup_map(
|
||||
model_names, internal_to_public
|
||||
).items()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def translate_listing(
|
||||
model_names: list[str],
|
||||
llm_router: "Router | None",
|
||||
general_settings: Mapping[str, object],
|
||||
) -> list[str]:
|
||||
"""Public-name view of `model_names` (the `response_id` of each listing
|
||||
entry). Sibling deployments sharing a public name collapse to one entry
|
||||
while preserving order; unmapped names pass through.
|
||||
"""
|
||||
return [
|
||||
entry[0]
|
||||
for entry in TeamModelNameTranslator.listing_entries(
|
||||
model_names, llm_router, general_settings
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def resolve_public_name(
|
||||
model_id: str,
|
||||
available_models: list[str],
|
||||
llm_router: "Router | None",
|
||||
general_settings: Mapping[str, object],
|
||||
) -> str:
|
||||
"""Resolve a public team name back to the internal routing key the router
|
||||
indexes by, so `GET /v1/models/{id}` accepts the name the listing returns.
|
||||
|
||||
Resolution is restricted to `available_models` (the caller's accessible
|
||||
set) so colliding public names across teams never resolve across an access
|
||||
boundary. Uses the same first-occurrence dedup as `listing_entries` so a
|
||||
public id advertised by `/v1/models` resolves to the same internal
|
||||
deployment that the listing's metadata was built from. Returns `model_id`
|
||||
unchanged when it is not an accessible public team name (already-internal
|
||||
names and globals pass through).
|
||||
"""
|
||||
internal_to_public = TeamModelNameTranslator.build_internal_to_public_map(
|
||||
llm_router, general_settings
|
||||
)
|
||||
if not internal_to_public:
|
||||
return model_id
|
||||
return TeamModelNameTranslator._response_to_lookup_map(
|
||||
available_models, internal_to_public
|
||||
).get(model_id, model_id)
|
||||
|
|
@ -10,6 +10,8 @@ from litellm.types.utils import SpecialEnums
|
|||
if TYPE_CHECKING:
|
||||
from fastapi import Request
|
||||
|
||||
from litellm.router import Router
|
||||
|
||||
|
||||
def _is_base64_encoded_unified_file_id(b64_uid: str) -> Union[str, Literal[False]]:
|
||||
# Ensure b64_uid is a string and not a mock object
|
||||
|
|
@ -296,6 +298,92 @@ def get_credentials_for_model(
|
|||
return credentials
|
||||
|
||||
|
||||
def get_team_provider_credentials(
|
||||
llm_router: Optional["Router"],
|
||||
team_models: List[str],
|
||||
custom_llm_provider: str,
|
||||
team_id: Optional[str] = None,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Resolve upstream credentials for a provider-scoped file operation
|
||||
(e.g. GET /v1/files), which doesn't pin a model.
|
||||
|
||||
Priority:
|
||||
1. The team's own (BYOK) deployment for this provider — a deployment whose
|
||||
``model_info.team_id`` matches ``team_id``. This keeps team-scoped listings
|
||||
on the team's own provider account/key instead of a shared global one.
|
||||
2. Fallback: any deployment the team is granted access to for this provider,
|
||||
expanding wildcard routes and the all-proxy-models sentinel.
|
||||
|
||||
Credential lookup is always scoped to the team's allowlist, so a team can
|
||||
never resolve a provider key for a deployment it isn't authorized to use.
|
||||
Returns None when the router is unavailable or no authorized deployment
|
||||
matches, so the caller can fall back to default credential resolution.
|
||||
"""
|
||||
if llm_router is None:
|
||||
return None
|
||||
|
||||
def _provider_credentials(model_id: str) -> Optional[dict]:
|
||||
credentials = llm_router.get_deployment_credentials_with_provider(
|
||||
model_id=model_id
|
||||
)
|
||||
if (
|
||||
credentials is not None
|
||||
and credentials.get("custom_llm_provider") == custom_llm_provider
|
||||
):
|
||||
return credentials
|
||||
return None
|
||||
|
||||
# 1. Prefer the team's own BYOK deployment, matched by model_info.team_id.
|
||||
if team_id is not None:
|
||||
for deployment in llm_router.model_list or []:
|
||||
model_info = deployment.get("model_info") or {}
|
||||
if model_info.get("team_id") != team_id:
|
||||
continue
|
||||
deployment_id = model_info.get("id")
|
||||
if deployment_id is None:
|
||||
continue
|
||||
credentials = _provider_credentials(deployment_id)
|
||||
if credentials is not None:
|
||||
return credentials
|
||||
|
||||
# 2. Fall back to deployments the team is allowed to access. The
|
||||
# all-proxy-models sentinel isn't expanded by get_complete_model_list, so
|
||||
# normalize it to an empty allowlist, which defers to the team-scoped
|
||||
# proxy model list. A team with a restricted allowlist (e.g. anthropic
|
||||
# only) therefore never resolves another provider's key.
|
||||
from litellm.proxy._types import SpecialModelNames
|
||||
from litellm.proxy.auth.model_checks import get_complete_model_list
|
||||
|
||||
grants_all_models = SpecialModelNames.all_proxy_models.value in team_models
|
||||
effective_team_models = [] if grants_all_models else team_models
|
||||
|
||||
proxy_model_list = llm_router.get_model_names(team_id=team_id)
|
||||
model_access_groups = llm_router.get_model_access_groups()
|
||||
models_to_try = list(
|
||||
dict.fromkeys(
|
||||
get_complete_model_list(
|
||||
key_models=[],
|
||||
team_models=effective_team_models,
|
||||
proxy_model_list=proxy_model_list,
|
||||
user_model=None,
|
||||
infer_model_from_keys=False,
|
||||
return_wildcard_routes=True,
|
||||
llm_router=llm_router,
|
||||
model_access_groups=model_access_groups,
|
||||
include_model_access_groups=True,
|
||||
team_id=team_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
for model_name in models_to_try:
|
||||
credentials = _provider_credentials(model_name)
|
||||
if credentials is not None:
|
||||
return credentials
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def prepare_data_with_credentials(
|
||||
data: dict,
|
||||
credentials: dict,
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
|
|||
encode_file_id_with_model,
|
||||
extract_file_creation_params,
|
||||
get_credentials_for_model,
|
||||
get_team_provider_credentials,
|
||||
handle_model_based_routing,
|
||||
prepare_data_with_credentials,
|
||||
)
|
||||
|
|
@ -1344,14 +1345,20 @@ async def list_files(
|
|||
status_code=400,
|
||||
detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']",
|
||||
)
|
||||
## Use router to list fine-tuning jobs for that model
|
||||
if llm_router is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail="LLM Router not initialized. Ensure models added to proxy.",
|
||||
)
|
||||
data["model"] = target_model_names_list[0]
|
||||
response = await llm_router.afile_list(
|
||||
credentials = get_credentials_for_model(
|
||||
llm_router=llm_router,
|
||||
model_id=target_model_names_list[0],
|
||||
operation_context="file list",
|
||||
)
|
||||
prepare_data_with_credentials(data=data, credentials=credentials)
|
||||
response = await litellm.afile_list(
|
||||
custom_llm_provider=credentials["custom_llm_provider"],
|
||||
purpose=purpose,
|
||||
**data,
|
||||
)
|
||||
else:
|
||||
|
|
@ -1363,6 +1370,18 @@ async def list_files(
|
|||
or "openai"
|
||||
)
|
||||
|
||||
# No model/target_model_names pinned: resolve upstream credentials from
|
||||
# the team's deployment for this provider so the call is authenticated
|
||||
# against the team's own account (e.g. the team's openai deployment).
|
||||
team_credentials = get_team_provider_credentials(
|
||||
llm_router=llm_router,
|
||||
team_models=user_api_key_dict.team_models or [],
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
team_id=user_api_key_dict.team_id,
|
||||
)
|
||||
if team_credentials is not None:
|
||||
prepare_data_with_credentials(data=data, credentials=team_credentials)
|
||||
|
||||
response = await litellm.afile_list(
|
||||
custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore
|
||||
)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import threading
|
|||
import time
|
||||
import traceback
|
||||
import warnings
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -302,6 +303,7 @@ from litellm.proxy.common_utils.load_config_utils import (
|
|||
get_config_file_contents_from_gcs,
|
||||
get_file_contents_from_s3,
|
||||
)
|
||||
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
|
||||
from litellm.proxy.common_utils.openai_endpoint_utils import (
|
||||
remove_sensitive_info_from_deployment,
|
||||
)
|
||||
|
|
@ -8202,6 +8204,7 @@ async def model_list(
|
|||
include_metadata: Optional[bool] = False,
|
||||
fallback_type: Optional[str] = None,
|
||||
scope: Optional[str] = None,
|
||||
healthy_only: Optional[bool] = False,
|
||||
):
|
||||
"""
|
||||
Use `/model/info` - to get detailed model information, example - pricing, mode, etc.
|
||||
|
|
@ -8215,9 +8218,20 @@ async def model_list(
|
|||
- scope: Optional scope parameter. Currently only accepts "expand".
|
||||
When scope=expand is passed, proxy admins, team admins, and org admins
|
||||
will receive all proxy models as if they are a proxy admin.
|
||||
- healthy_only: When true, hide models whose backing deployments are all marked
|
||||
unhealthy by background health checks. Requires
|
||||
`background_health_checks: true` in general_settings; without
|
||||
health state the listing is returned unfiltered (fail open).
|
||||
Models expanded from wildcard routes (e.g. `openai/*`) are not
|
||||
filtered, and nothing is hidden when `allowed_fails_policy` is
|
||||
configured (cooldown remains the sole exclusion mechanism).
|
||||
Hiding is presentation-only: a hidden model can still be
|
||||
called directly.
|
||||
"""
|
||||
global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
|
||||
|
||||
settings = cast(dict[str, object], general_settings) # any-ok: legacy settings
|
||||
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
_user_has_admin_privileges,
|
||||
)
|
||||
|
|
@ -8248,6 +8262,19 @@ async def model_list(
|
|||
llm_router.get_fully_blocked_model_names() if llm_router is not None else set()
|
||||
)
|
||||
|
||||
# Opt-in: also hide models whose deployments are all unhealthy per background
|
||||
# health checks. Empty when health state is unavailable or stale (fail open).
|
||||
unhealthy_names: Set[str] = set()
|
||||
if healthy_only and llm_router is not None:
|
||||
unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names()
|
||||
if not unhealthy_names:
|
||||
verbose_proxy_logger.debug(
|
||||
"healthy_only=true but no unhealthy deployment state is available "
|
||||
"(requires background_health_checks); returning unfiltered model list"
|
||||
)
|
||||
|
||||
hidden_names = blocked_names | unhealthy_names
|
||||
|
||||
# If scope=expand and user has admin privileges, return all proxy models
|
||||
if should_expand_scope:
|
||||
# Get all proxy models as if user is a proxy admin
|
||||
|
|
@ -8280,20 +8307,25 @@ async def model_list(
|
|||
only_model_access_groups=only_model_access_groups or False,
|
||||
)
|
||||
|
||||
# Hide paused models from the public listing (admins manage them via /model/info)
|
||||
if blocked_names:
|
||||
all_models = [m for m in all_models if m not in blocked_names]
|
||||
# Hide paused/unhealthy models from the public listing
|
||||
if hidden_names:
|
||||
all_models = [m for m in all_models if m not in hidden_names]
|
||||
|
||||
# Build response data with all proxy models
|
||||
# Surface the public team name by default; legacy internal keys via flag.
|
||||
# The internal routing key drives the metadata/fallback lookup, while the
|
||||
# public name is what the client sees as the model id.
|
||||
model_data = []
|
||||
for model in all_models:
|
||||
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(
|
||||
all_models, llm_router, settings
|
||||
):
|
||||
model_info = create_model_info_response(
|
||||
model_id=model,
|
||||
model_id=lookup_id,
|
||||
provider="openai",
|
||||
include_metadata=include_metadata or False,
|
||||
fallback_type=fallback_type,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
model_info["id"] = response_id
|
||||
model_data.append(model_info)
|
||||
|
||||
return dict(
|
||||
|
|
@ -8317,20 +8349,25 @@ async def model_list(
|
|||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
|
||||
# Hide paused models from the public listing (admins manage them via /model/info)
|
||||
if blocked_names:
|
||||
all_models = [m for m in all_models if m not in blocked_names]
|
||||
# Hide paused/unhealthy models from the public listing
|
||||
if hidden_names:
|
||||
all_models = [m for m in all_models if m not in hidden_names]
|
||||
|
||||
# Build response data
|
||||
# Surface the public team name by default; legacy internal keys via flag.
|
||||
# The internal routing key drives the metadata/fallback lookup, while the
|
||||
# public name is what the client sees as the model id.
|
||||
model_data = []
|
||||
for model in all_models:
|
||||
for response_id, lookup_id in TeamModelNameTranslator.listing_entries(
|
||||
all_models, llm_router, settings
|
||||
):
|
||||
model_info = create_model_info_response(
|
||||
model_id=model,
|
||||
model_id=lookup_id,
|
||||
provider="openai",
|
||||
include_metadata=include_metadata or False,
|
||||
fallback_type=fallback_type,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
model_info["id"] = response_id
|
||||
model_data.append(model_info)
|
||||
|
||||
return dict(
|
||||
|
|
@ -8352,6 +8389,8 @@ async def model_list(
|
|||
async def model_info(
|
||||
model_id: str,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
team_id: Optional[str] = None,
|
||||
healthy_only: Optional[bool] = False,
|
||||
):
|
||||
"""
|
||||
Retrieve information about a specific model accessible to your API key.
|
||||
|
|
@ -8361,16 +8400,21 @@ async def model_info(
|
|||
|
||||
Follows OpenAI API specification for individual model retrieval.
|
||||
https://platform.openai.com/docs/api-reference/models/retrieve
|
||||
|
||||
Query parameters mirror `/v1/models` so the same caller context (team
|
||||
scoping, health filtering, paused deployments) drives both endpoints; the
|
||||
listing's public id must resolve to the same internal deployment here.
|
||||
"""
|
||||
global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
|
||||
|
||||
settings = cast(dict[str, object], general_settings) # any-ok: legacy settings
|
||||
|
||||
from litellm.proxy.utils import (
|
||||
create_model_info_response,
|
||||
get_available_models_for_user,
|
||||
validate_model_access,
|
||||
)
|
||||
|
||||
# Get available models for the user
|
||||
all_models = await get_available_models_for_user(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
llm_router=llm_router,
|
||||
|
|
@ -8378,21 +8422,43 @@ async def model_info(
|
|||
user_model=user_model,
|
||||
prisma_client=prisma_client,
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
team_id=None,
|
||||
team_id=team_id,
|
||||
include_model_access_groups=False,
|
||||
only_model_access_groups=False,
|
||||
return_wildcard_routes=False,
|
||||
user_api_key_cache=user_api_key_cache,
|
||||
)
|
||||
|
||||
# Mirror /v1/models' visibility filter so first-occurrence resolution
|
||||
# cannot land on a deployment the listing had hidden.
|
||||
blocked_names = (
|
||||
llm_router.get_fully_blocked_model_names() if llm_router is not None else set()
|
||||
)
|
||||
unhealthy_names: set[str] = set()
|
||||
if healthy_only and llm_router is not None:
|
||||
unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names()
|
||||
hidden_names = blocked_names | unhealthy_names
|
||||
if hidden_names:
|
||||
all_models = [m for m in all_models if m not in hidden_names]
|
||||
|
||||
internal_to_public = TeamModelNameTranslator.build_internal_to_public_map(
|
||||
llm_router, settings
|
||||
)
|
||||
resolved_model_id = TeamModelNameTranslator.resolve_public_name(
|
||||
model_id=model_id,
|
||||
available_models=all_models,
|
||||
llm_router=llm_router,
|
||||
general_settings=settings,
|
||||
)
|
||||
|
||||
# Validate that the requested model is accessible
|
||||
validate_model_access(model_id=model_id, available_models=all_models)
|
||||
validate_model_access(model_id=resolved_model_id, available_models=all_models)
|
||||
|
||||
# Get provider information from the router deployment
|
||||
if llm_router is None:
|
||||
raise HTTPException(status_code=500, detail="Router not initialized")
|
||||
|
||||
deployment = llm_router.get_deployment_by_model_group_name(model_id)
|
||||
deployment = llm_router.get_deployment_by_model_group_name(resolved_model_id)
|
||||
if deployment is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
|
|
@ -8402,9 +8468,9 @@ async def model_info(
|
|||
# Use the actual litellm model from the deployment to get provider info
|
||||
_, provider, _, _ = litellm.get_llm_provider(model=deployment.litellm_params.model)
|
||||
|
||||
# Return the model information in the same format as the list endpoint
|
||||
response_id = internal_to_public.get(resolved_model_id, model_id)
|
||||
return create_model_info_response(
|
||||
model_id=model_id,
|
||||
model_id=response_id,
|
||||
provider=provider,
|
||||
include_metadata=False,
|
||||
fallback_type=None,
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ from litellm.proxy._types import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.proxy.model_listing import ModelInfoResponse
|
||||
from litellm.types.utils import CallTypes, CallTypesLiteral
|
||||
|
||||
try:
|
||||
|
|
@ -6231,56 +6232,39 @@ def create_model_info_response(
|
|||
include_metadata: bool = False,
|
||||
fallback_type: Optional[str] = None,
|
||||
llm_router: Optional["Router"] = None,
|
||||
) -> dict:
|
||||
) -> ModelInfoResponse:
|
||||
"""
|
||||
Create a standardized model info response.
|
||||
Create a standardized OpenAI-compatible model object.
|
||||
|
||||
Args:
|
||||
model_id: The model ID
|
||||
provider: The model provider
|
||||
include_metadata: Whether to include metadata
|
||||
fallback_type: Type of fallbacks to include
|
||||
llm_router: LiteLLM router instance
|
||||
|
||||
Returns:
|
||||
Dictionary containing model information
|
||||
When include_metadata is true, attaches the model's configured fallbacks
|
||||
(resolved via the router under fallback_type, defaulting to "general").
|
||||
Raises HTTPException(400) for an unknown fallback_type.
|
||||
"""
|
||||
from litellm.proxy.auth.model_checks import get_all_fallbacks
|
||||
|
||||
model_info = {
|
||||
base: ModelInfoResponse = {
|
||||
"id": model_id,
|
||||
"object": "model",
|
||||
"created": DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
"owned_by": provider,
|
||||
}
|
||||
if not include_metadata:
|
||||
return base
|
||||
|
||||
# Add metadata if requested
|
||||
if include_metadata:
|
||||
metadata = {}
|
||||
|
||||
# Default fallback_type to "general" if include_metadata is true
|
||||
effective_fallback_type = (
|
||||
fallback_type if fallback_type is not None else "general"
|
||||
effective_fallback_type = fallback_type if fallback_type is not None else "general"
|
||||
valid_fallback_types = ("general", "context_window", "content_policy")
|
||||
if effective_fallback_type not in valid_fallback_types:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid fallback_type. Must be one of: {list(valid_fallback_types)}",
|
||||
)
|
||||
|
||||
# Validate fallback_type
|
||||
valid_fallback_types = ["general", "context_window", "content_policy"]
|
||||
if effective_fallback_type not in valid_fallback_types:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Invalid fallback_type. Must be one of: {valid_fallback_types}",
|
||||
)
|
||||
|
||||
fallbacks = get_all_fallbacks(
|
||||
model=model_id,
|
||||
llm_router=llm_router,
|
||||
fallback_type=effective_fallback_type,
|
||||
)
|
||||
metadata["fallbacks"] = fallbacks
|
||||
|
||||
model_info["metadata"] = metadata
|
||||
|
||||
return model_info
|
||||
fallbacks = get_all_fallbacks(
|
||||
model=model_id,
|
||||
llm_router=llm_router,
|
||||
fallback_type=effective_fallback_type,
|
||||
)
|
||||
return {**base, "metadata": {"fallbacks": fallbacks}}
|
||||
|
||||
|
||||
def validate_model_access(
|
||||
|
|
|
|||
|
|
@ -9948,6 +9948,63 @@ class Router:
|
|||
name for name, fully_blocked in blocked_by_name.items() if fully_blocked
|
||||
}
|
||||
|
||||
async def async_get_fully_unhealthy_model_names(self) -> Set[str]:
|
||||
"""
|
||||
Returns the set of model names where every backing deployment is currently
|
||||
marked unhealthy by background health checks (and the health state is not stale).
|
||||
|
||||
Used by `/v1/models?healthy_only=true` to hide models that cannot serve any
|
||||
request. A model with at least one healthy (or unknown-health) deployment
|
||||
remains visible. Returns an empty set when no health state is available, so
|
||||
callers fail open to the unfiltered listing.
|
||||
|
||||
Notes:
|
||||
- Mirrors `_async_filter_health_check_unhealthy_deployments`: when
|
||||
`allowed_fails_policy` is set, cooldown is the sole routing exclusion
|
||||
mechanism, so nothing is hidden here either.
|
||||
- Team-specific public model names (`team_public_model_name`) are
|
||||
aggregated alongside `model_name`, so team aliases of fully-unhealthy
|
||||
deployments are hidden too (unlike `get_fully_blocked_model_names`,
|
||||
which matches `model_name` only).
|
||||
- Wildcard routes (e.g. `openai/*`) are matched by their literal
|
||||
deployment name only; models expanded from a wildcard route are not
|
||||
hidden (fail open).
|
||||
- Intentionally diverges from the routing-time safety net (which
|
||||
bypasses the health filter when every candidate is unhealthy and
|
||||
still attempts the request): hiding here is presentation-only —
|
||||
it answers "should this model be advertised?", not "should a
|
||||
request for it still be attempted?". A hidden model can still be
|
||||
called directly.
|
||||
"""
|
||||
if self.allowed_fails_policy is not None:
|
||||
return set()
|
||||
unhealthy_ids = (
|
||||
await self.health_state_cache.async_get_unhealthy_deployment_ids()
|
||||
)
|
||||
if not unhealthy_ids:
|
||||
return set()
|
||||
deployments = self.get_model_list() or []
|
||||
unhealthy_by_name: Dict[str, bool] = {}
|
||||
for deployment in deployments:
|
||||
model_info = deployment.get("model_info") or {}
|
||||
names = [deployment.get("model_name") or ""]
|
||||
team_public_model_name = model_info.get("team_public_model_name")
|
||||
if team_public_model_name:
|
||||
names.append(team_public_model_name)
|
||||
is_unhealthy = model_info.get("id") in unhealthy_ids
|
||||
for name in names:
|
||||
if not name:
|
||||
continue
|
||||
if name in unhealthy_by_name:
|
||||
unhealthy_by_name[name] = unhealthy_by_name[name] and is_unhealthy
|
||||
else:
|
||||
unhealthy_by_name[name] = is_unhealthy
|
||||
return {
|
||||
name
|
||||
for name, fully_unhealthy in unhealthy_by_name.items()
|
||||
if fully_unhealthy
|
||||
}
|
||||
|
||||
def _get_team_specific_model(
|
||||
self, deployment: DeploymentTypedDict, team_id: Optional[str] = None
|
||||
) -> Optional[str]:
|
||||
|
|
|
|||
21
litellm/types/proxy/model_listing.py
Normal file
21
litellm/types/proxy/model_listing.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
"""Response types for the model listing/retrieve endpoints (/v1/models, /models)."""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from typing_extensions import NotRequired, TypedDict
|
||||
|
||||
|
||||
class ModelInfoMetadata(TypedDict):
|
||||
fallbacks: list[str]
|
||||
|
||||
|
||||
class ModelInfoResponse(TypedDict):
|
||||
"""OpenAI-compatible model object. `metadata` is present only when the
|
||||
endpoint is called with include_metadata=true.
|
||||
"""
|
||||
|
||||
id: str
|
||||
object: Literal["model"]
|
||||
created: int
|
||||
owned_by: str
|
||||
metadata: NotRequired[ModelInfoMetadata]
|
||||
|
|
@ -3635,6 +3635,7 @@ class SpecialEnums(Enum):
|
|||
class ServiceTier(Enum):
|
||||
"""Enum for service tier types used in cost calculations."""
|
||||
|
||||
AUTO = "auto"
|
||||
FLEX = "flex"
|
||||
PRIORITY = "priority"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm"
|
||||
version = "1.89.1"
|
||||
version = "1.89.2"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10, <3.14"
|
||||
|
|
@ -264,7 +264,7 @@ source-exclude = [
|
|||
profile = "black"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.89.1"
|
||||
version = "1.89.2"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -906,7 +906,10 @@ class BaseLLMChatTest(ABC):
|
|||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://www.gstatic.com/webp/gallery/1.webp",
|
||||
# sha-pinned in-repo logo via jsdelivr; gstatic's
|
||||
# robots.txt blocks server-side fetchers (e.g.
|
||||
# Anthropic), which 400s the request.
|
||||
"url": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0/ui/litellm-dashboard/public/assets/logos/litellm_logo.jpg",
|
||||
"detail": detail,
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -410,6 +410,90 @@ def test_emitter_without_call_id_is_not_deduped():
|
|||
assert len(exporter.get_finished_spans()) == 2
|
||||
|
||||
|
||||
def _emit_error_span(message, error_type="litellm.APIError"):
|
||||
from litellm.integrations.otel.emitter import SpanEmitter
|
||||
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory")
|
||||
provider, exporter = providers.in_memory_provider(cfg)
|
||||
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
|
||||
data = LLMCallSpanData(
|
||||
operation=GenAIOperation.CHAT,
|
||||
provider="openai",
|
||||
request_model="gpt-4o",
|
||||
response_model=None,
|
||||
response_id=None,
|
||||
request_params=LLMRequestParams(),
|
||||
usage=LLMUsage(),
|
||||
finish_reasons=(),
|
||||
error=SpanError(error_type=error_type, message=message),
|
||||
response_cost=None,
|
||||
server=None,
|
||||
identity=RequestIdentity(call_id=None),
|
||||
)
|
||||
engine.emit(SpanRole.LLM_CALL, data)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
return span
|
||||
|
||||
|
||||
def _exception_event(span):
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent
|
||||
|
||||
events = [e for e in span.events if e.name == ExceptionEvent.NAME]
|
||||
assert len(events) == 1, "expected exactly one exception event"
|
||||
return events[0]
|
||||
|
||||
|
||||
def test_error_message_recorded_as_full_exception_event_untruncated():
|
||||
"""Regression for the Elasticsearch keyword/ignore_above:1024 truncation.
|
||||
|
||||
A long error message must survive intact on the standard ``exception``
|
||||
event under ``exception.message`` — not get dropped onto a bare string
|
||||
attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK
|
||||
must not truncate it either, so a 5000-char message stays 5000 chars.
|
||||
"""
|
||||
from litellm.integrations.otel.model.semconv import Error, ExceptionEvent
|
||||
|
||||
long_message = "boom: " + "x" * 5000
|
||||
span = _emit_error_span(long_message, error_type="litellm.APIError")
|
||||
|
||||
event = _exception_event(span)
|
||||
assert event.attributes[ExceptionEvent.MESSAGE] == long_message
|
||||
assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024
|
||||
assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError"
|
||||
|
||||
# error.type stays a low-cardinality attribute; the message does NOT become a
|
||||
# bare string attribute (which is what got truncated).
|
||||
assert span.attributes[Error.TYPE] == "litellm.APIError"
|
||||
assert ExceptionEvent.MESSAGE not in span.attributes
|
||||
assert span.status.description == long_message
|
||||
|
||||
|
||||
def test_success_span_records_no_exception_event():
|
||||
from litellm.integrations.otel.emitter import SpanEmitter
|
||||
from litellm.integrations.otel.model.semconv import ExceptionEvent
|
||||
|
||||
cfg = OpenTelemetryV2Config(exporter="in_memory")
|
||||
provider, exporter = providers.in_memory_provider(cfg)
|
||||
engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg)
|
||||
data = LLMCallSpanData(
|
||||
operation=GenAIOperation.CHAT,
|
||||
provider="openai",
|
||||
request_model="gpt-4o",
|
||||
response_model="gpt-4o",
|
||||
response_id="resp-1",
|
||||
request_params=LLMRequestParams(),
|
||||
usage=LLMUsage(),
|
||||
finish_reasons=("stop",),
|
||||
error=None,
|
||||
response_cost=None,
|
||||
server=None,
|
||||
identity=RequestIdentity(call_id=None),
|
||||
)
|
||||
engine.emit(SpanRole.LLM_CALL, data)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert all(e.name != ExceptionEvent.NAME for e in span.events)
|
||||
|
||||
|
||||
# --- service taxonomy: which calls become spans, and of what kind ----------- #
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3702,6 +3702,39 @@ def test_fast_mode_with_inference_geo():
|
|||
assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10
|
||||
|
||||
|
||||
def test_calculate_usage_captures_service_tier():
|
||||
"""
|
||||
Anthropic returns the assigned service tier on the response usage object
|
||||
(e.g. ``"priority"``). It must be surfaced on the Usage object so it is
|
||||
visible in logs and used to select tier-specific pricing.
|
||||
"""
|
||||
config = AnthropicConfig()
|
||||
|
||||
usage_object = {
|
||||
"input_tokens": 410,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"cache_read_input_tokens": 0,
|
||||
"output_tokens": 585,
|
||||
"service_tier": "priority",
|
||||
}
|
||||
|
||||
usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None)
|
||||
|
||||
assert usage.service_tier == "priority"
|
||||
|
||||
|
||||
def test_calculate_usage_service_tier_defaults_to_none():
|
||||
"""A response without a service tier must not invent one."""
|
||||
config = AnthropicConfig()
|
||||
|
||||
usage = config.calculate_usage(
|
||||
usage_object={"input_tokens": 10, "output_tokens": 5},
|
||||
reasoning_content=None,
|
||||
)
|
||||
|
||||
assert usage.service_tier is None
|
||||
|
||||
|
||||
def test_fast_mode_parameter_in_supported_params():
|
||||
"""
|
||||
Test that 'speed' is in the list of supported OpenAI params.
|
||||
|
|
|
|||
|
|
@ -1400,6 +1400,65 @@ def test_rag_routes_accessible_to_internal_user_viewer():
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"route",
|
||||
[
|
||||
"/vector_stores/vs_123",
|
||||
"/v1/vector_stores/vs_123",
|
||||
"/vector_stores/vs_123/search",
|
||||
"/v1/vector_stores/vs_123/search",
|
||||
"/vector_stores/vs_123/files",
|
||||
"/v1/vector_stores/vs_123/files",
|
||||
],
|
||||
)
|
||||
def test_vector_store_routes_are_llm_api_routes(route):
|
||||
"""Retrieve/update/delete on a single vector store must classify as LLM API routes.
|
||||
|
||||
Regression for the missing bare `/v1/vector_stores/{vector_store_id}` entry in
|
||||
`openai_routes` that left retrieve/update/delete blocked for internal roles
|
||||
while `/search` and `/files` sub-routes worked.
|
||||
"""
|
||||
|
||||
assert RouteChecks.is_llm_api_route(route) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"user_role",
|
||||
[
|
||||
LitellmUserRoles.INTERNAL_USER.value,
|
||||
LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"method, route",
|
||||
[
|
||||
("GET", "/v1/vector_stores/vs_123"),
|
||||
("POST", "/v1/vector_stores/vs_123"),
|
||||
("DELETE", "/v1/vector_stores/vs_123"),
|
||||
],
|
||||
)
|
||||
def test_vector_store_crud_accessible_to_internal_roles(user_role, method, route):
|
||||
"""Internal user and internal viewer must reach vector store retrieve/update/delete.
|
||||
|
||||
Object-level access is still gated by `assert_user_can_access_vector_store`;
|
||||
this only verifies the route gate no longer 403s these roles.
|
||||
"""
|
||||
|
||||
valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role)
|
||||
request = MagicMock(spec=Request)
|
||||
request.method = method
|
||||
request.query_params = {}
|
||||
|
||||
RouteChecks.non_proxy_admin_allowed_routes_check(
|
||||
user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role),
|
||||
_user_role=user_role,
|
||||
route=route,
|
||||
request=request,
|
||||
valid_token=valid_token,
|
||||
request_data={},
|
||||
)
|
||||
|
||||
|
||||
def test_videos_route_accessible_to_internal_users():
|
||||
"""
|
||||
Test that internal users can access the videos routes.
|
||||
|
|
|
|||
|
|
@ -1873,3 +1873,329 @@ def test_get_file_content_non_openai_provider_skips_streaming_handler(
|
|||
assert "stream" not in captured_kwargs
|
||||
mock_streaming_response.assert_not_awaited()
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
||||
|
||||
def test_list_files_resolves_wildcard_deployment_credentials(
|
||||
mocker: MockerFixture, monkeypatch
|
||||
):
|
||||
"""
|
||||
GET /v1/files?target_model_names=<model> must resolve the upstream api_key
|
||||
from the matching (wildcard) deployment. Regression for the path routing
|
||||
through llm_router.afile_list(model=...), which reached OpenAI without an
|
||||
api_key and failed with "api_key client option must be set".
|
||||
"""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
wildcard_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "wildcard-openai-key",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router)
|
||||
proxy_logging_obj.update_request_status = mocker.AsyncMock()
|
||||
proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[])
|
||||
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
async def _mock_afile_list(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(litellm, "afile_list", _mock_afile_list)
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
user_id="test-user",
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.get(
|
||||
"/v1/files?target_model_names=gpt-4o",
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert captured_kwargs.get("api_key") == "wildcard-openai-key"
|
||||
assert captured_kwargs.get("custom_llm_provider") == "openai"
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
||||
|
||||
def test_list_files_without_target_model_names_uses_team_openai_deployment(
|
||||
mocker: MockerFixture, monkeypatch
|
||||
):
|
||||
"""
|
||||
Plain GET /v1/files (no target_model_names) must resolve the upstream openai
|
||||
api_key from the team's openai deployment instead of falling through to a
|
||||
keyless OpenAI client. Regression for "api_key client option must be set".
|
||||
"""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
wildcard_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "team-openai-key",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router)
|
||||
proxy_logging_obj.update_request_status = mocker.AsyncMock()
|
||||
proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[])
|
||||
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
async def _mock_afile_list(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(litellm, "afile_list", _mock_afile_list)
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
user_id="test-user",
|
||||
team_id="test-team",
|
||||
team_models=["openai/*"],
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.get(
|
||||
"/v1/files",
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert captured_kwargs.get("api_key") == "team-openai-key"
|
||||
assert captured_kwargs.get("custom_llm_provider") == "openai"
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
||||
|
||||
def test_list_files_restricted_team_does_not_leak_global_openai_credentials(
|
||||
mocker: MockerFixture, monkeypatch
|
||||
):
|
||||
"""
|
||||
A team whose allowlist only grants anthropic must NOT resolve a global
|
||||
openai deployment's api_key for plain GET /v1/files. Regression for the
|
||||
last-resort scan that ignored team access control.
|
||||
"""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "global-openai-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-opus-4-6",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-opus-4-6",
|
||||
"api_key": "anthropic-key",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
|
||||
proxy_logging_obj.update_request_status = mocker.AsyncMock()
|
||||
proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[])
|
||||
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
async def _mock_afile_list(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(litellm, "afile_list", _mock_afile_list)
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
user_id="test-user",
|
||||
team_id="anthropic-only-team",
|
||||
team_models=["claude-opus-4-6"],
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.get(
|
||||
"/v1/files",
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert captured_kwargs.get("api_key") != "global-openai-key"
|
||||
|
||||
|
||||
def test_list_files_prefers_team_byok_over_global_openai_deployment(
|
||||
mocker: MockerFixture, monkeypatch
|
||||
):
|
||||
"""
|
||||
When a team has its own BYOK openai deployment (model_info.team_id set), plain
|
||||
GET /v1/files must use the team's key, not a shared/global openai deployment.
|
||||
"""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles
|
||||
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "global-openai-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "team-gpt-4o",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "team-byok-openai-key",
|
||||
},
|
||||
"model_info": {
|
||||
"id": "team-byok-deployment-id",
|
||||
"team_id": "test-team",
|
||||
"team_public_model_name": "team-gpt-4o",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
|
||||
proxy_logging_obj.update_request_status = mocker.AsyncMock()
|
||||
proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[])
|
||||
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
async def _mock_afile_list(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(litellm, "afile_list", _mock_afile_list)
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
user_id="test-user",
|
||||
team_id="test-team",
|
||||
team_models=["team-gpt-4o"],
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.get(
|
||||
"/v1/files",
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert captured_kwargs.get("api_key") == "team-byok-openai-key"
|
||||
assert captured_kwargs.get("custom_llm_provider") == "openai"
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
||||
|
||||
def test_list_files_with_all_proxy_models_team_uses_openai_deployment(
|
||||
mocker: MockerFixture, monkeypatch
|
||||
):
|
||||
"""
|
||||
Teams with all-proxy-models (or empty models) must still resolve openai
|
||||
credentials for plain GET /v1/files.
|
||||
"""
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles, SpecialModelNames
|
||||
|
||||
wildcard_router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {
|
||||
"model": "openai/*",
|
||||
"api_key": "team-openai-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "claude-opus-4-6",
|
||||
"litellm_params": {
|
||||
"model": "anthropic/claude-opus-4-6",
|
||||
"api_key": "anthropic-key",
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router)
|
||||
proxy_logging_obj.update_request_status = mocker.AsyncMock()
|
||||
proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[])
|
||||
proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock()
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
|
||||
async def _mock_afile_list(**kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(litellm, "afile_list", _mock_afile_list)
|
||||
|
||||
app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(
|
||||
api_key="test-key",
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
user_id="test-user",
|
||||
team_id="test-team",
|
||||
team_models=[SpecialModelNames.all_proxy_models.value],
|
||||
)
|
||||
|
||||
try:
|
||||
response = client.get(
|
||||
"/v1/files",
|
||||
headers={"Authorization": "Bearer test-key"},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.pop(ps.user_api_key_auth, None)
|
||||
|
||||
assert response.status_code == 200, response.text
|
||||
assert captured_kwargs.get("api_key") == "team-openai-key"
|
||||
assert captured_kwargs.get("custom_llm_provider") == "openai"
|
||||
proxy_logging_obj.post_call_failure_hook.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import pytest
|
|||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator
|
||||
from litellm.proxy.proxy_server import (
|
||||
_get_proxy_model_info,
|
||||
_translate_model_name_for_response,
|
||||
|
|
@ -593,3 +594,664 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey
|
|||
team_filter.assert_awaited_once()
|
||||
assert team_filter.await_args.kwargs["team_id"] == "other-team"
|
||||
assert team_filter.await_args.kwargs["all_models"] == [team_row]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_models_translates_team_model_for_access_group_key(monkeypatch):
|
||||
"""Regression (#28382 sibling leak): a virtual key whose model access group
|
||||
resolves to a team BYOK deployment must list the PUBLIC name in /v1/models,
|
||||
not the internal routing key model_name_{team_id}_{uuid}.
|
||||
|
||||
The /model/info read-path fix did not cover /v1/models, which builds from
|
||||
bare model-name strings via access-group expansion.
|
||||
"""
|
||||
team_dep = {
|
||||
"model_name": "model_name_teamX_uuid9",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {
|
||||
"id": "id1",
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
"access_groups": ["grp-a"],
|
||||
},
|
||||
}
|
||||
router = MagicMock()
|
||||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "user_model", None)
|
||||
# Default behavior: listing surfaces public names.
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
|
||||
# virtual key granted access via the access group (no team membership)
|
||||
key = UserAPIKeyAuth(
|
||||
user_id="u", api_key="sk-test", models=["grp-a"], team_models=[]
|
||||
)
|
||||
resp = await ps.model_list(user_api_key_dict=key)
|
||||
|
||||
ids = [d["id"] for d in resp["data"]]
|
||||
assert "tushar-gpt-4.1" in ids
|
||||
assert "model_name_teamX_uuid9" not in ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_models_keeps_internal_names_when_public_name_flag_disabled(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Compatibility override: /v1/models can still list the internal routing
|
||||
name for consumers that scripted against those ids. Translation is enabled
|
||||
by default and disabled via general_settings['use_team_public_model_name'].
|
||||
"""
|
||||
team_dep = {
|
||||
"model_name": "model_name_teamX_uuid9",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {
|
||||
"id": "id1",
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
"access_groups": ["grp-a"],
|
||||
},
|
||||
}
|
||||
router = MagicMock()
|
||||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "user_model", None)
|
||||
monkeypatch.setattr(ps, "general_settings", {"use_team_public_model_name": False})
|
||||
|
||||
key = UserAPIKeyAuth(
|
||||
user_id="u", api_key="sk-test", models=["grp-a"], team_models=[]
|
||||
)
|
||||
resp = await ps.model_list(user_api_key_dict=key)
|
||||
|
||||
ids = [d["id"] for d in resp["data"]]
|
||||
assert "model_name_teamX_uuid9" in ids # internal id preserved (backward-compat)
|
||||
assert "tushar-gpt-4.1" not in ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_models_translates_team_model_with_metadata(monkeypatch):
|
||||
"""include_metadata=true must build metadata for the public model id."""
|
||||
team_dep = {
|
||||
"model_name": "model_name_teamX_uuid9",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {
|
||||
"id": "id1",
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
"access_groups": ["grp-a"],
|
||||
},
|
||||
}
|
||||
router = MagicMock()
|
||||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "user_model", None)
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
|
||||
key = UserAPIKeyAuth(
|
||||
user_id="u", api_key="sk-test", models=["grp-a"], team_models=[]
|
||||
)
|
||||
resp = await ps.model_list(user_api_key_dict=key, include_metadata=True)
|
||||
|
||||
assert resp["data"] == [
|
||||
{
|
||||
"id": "tushar-gpt-4.1",
|
||||
"object": "model",
|
||||
"created": 1677610602,
|
||||
"owned_by": "openai",
|
||||
"metadata": {"fallbacks": []},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch):
|
||||
"""Regression: with include_metadata=true, fallbacks configured for a team
|
||||
model under its internal routing key must still surface. The metadata lookup
|
||||
has to run against the internal name, not the translated public name (which
|
||||
the router's fallback config never keys on) -- otherwise fallbacks silently
|
||||
drop to []."""
|
||||
team_dep = {
|
||||
"model_name": "model_name_teamX_uuid9",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {
|
||||
"id": "id1",
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
"access_groups": ["grp-a"],
|
||||
},
|
||||
}
|
||||
router = MagicMock()
|
||||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.model_list = [team_dep]
|
||||
router.get_model_list.return_value = [team_dep]
|
||||
# Fallbacks are keyed on the internal routing name, as the router stores them.
|
||||
router.fallbacks = [{"model_name_teamX_uuid9": ["gpt-4o-backup"]}]
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "user_model", None)
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
|
||||
key = UserAPIKeyAuth(
|
||||
user_id="u", api_key="sk-test", models=["grp-a"], team_models=[]
|
||||
)
|
||||
resp = await ps.model_list(user_api_key_dict=key, include_metadata=True)
|
||||
|
||||
assert resp["data"] == [
|
||||
{
|
||||
"id": "tushar-gpt-4.1",
|
||||
"object": "model",
|
||||
"created": 1677610602,
|
||||
"owned_by": "openai",
|
||||
"metadata": {"fallbacks": ["gpt-4o-backup"]},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch):
|
||||
"""Regression: two teams can publish the same team_public_model_name. With
|
||||
include_metadata=true a caller scoped to teamX must see teamX's fallbacks for
|
||||
the shared public name, never teamY's. The metadata lookup has to stay within
|
||||
the caller's accessible models; resolving the public name through a router-wide
|
||||
reverse map could point it at another team's internal routing key."""
|
||||
team_x = {
|
||||
"model_name": "model_name_teamX_uuid9",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {
|
||||
"id": "idX",
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
"access_groups": ["grp-a"],
|
||||
},
|
||||
}
|
||||
team_y = {
|
||||
"model_name": "model_name_teamY_uuidZ",
|
||||
"litellm_params": {"model": "azure/gpt-4.1"},
|
||||
"model_info": {
|
||||
"id": "idY",
|
||||
"team_id": "teamY",
|
||||
"team_public_model_name": "tushar-gpt-4.1", # same public name, other team
|
||||
},
|
||||
}
|
||||
router = MagicMock()
|
||||
router.get_model_names.return_value = ["model_name_teamX_uuid9"]
|
||||
router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]}
|
||||
router.get_fully_blocked_model_names.return_value = set()
|
||||
router.model_list = [team_x, team_y]
|
||||
router.get_model_list.return_value = [team_x, team_y]
|
||||
router.fallbacks = [
|
||||
{"model_name_teamX_uuid9": ["teamX-backup"]},
|
||||
{"model_name_teamY_uuidZ": ["teamY-backup"]},
|
||||
]
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "user_model", None)
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
|
||||
key = UserAPIKeyAuth(
|
||||
user_id="u", api_key="sk-test", models=["grp-a"], team_models=[]
|
||||
)
|
||||
resp = await ps.model_list(user_api_key_dict=key, include_metadata=True)
|
||||
|
||||
assert resp["data"] == [
|
||||
{
|
||||
"id": "tushar-gpt-4.1",
|
||||
"object": "model",
|
||||
"created": 1677610602,
|
||||
"owned_by": "openai",
|
||||
"metadata": {"fallbacks": ["teamX-backup"]},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_translate_team_model_names_for_listing_swaps_and_dedupes():
|
||||
"""Internal team routing keys -> public name; sibling deployments sharing a
|
||||
public name collapse to one entry (order preserved); globals untouched."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidB", # sibling: same public name
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
},
|
||||
{"model_name": "gpt-4o", "model_info": {"db_model": False}},
|
||||
]
|
||||
|
||||
out = TeamModelNameTranslator.translate_listing(
|
||||
["model_name_teamX_uuidA", "model_name_teamX_uuidB", "gpt-4o"],
|
||||
router,
|
||||
{},
|
||||
)
|
||||
assert out == ["tushar-gpt-4.1", "gpt-4o"]
|
||||
|
||||
|
||||
def test_listing_entries_keep_internal_lookup_id_for_team_rows():
|
||||
"""`listing_entries` returns (public response id, internal lookup id) so the
|
||||
response shows the public name while metadata lookups keep the routing key.
|
||||
Sibling deployments collapse to one entry; globals map to themselves."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidB", # sibling: same public name
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
},
|
||||
{"model_name": "gpt-4o", "model_info": {"db_model": False}},
|
||||
]
|
||||
|
||||
entries = TeamModelNameTranslator.listing_entries(
|
||||
["model_name_teamX_uuidA", "model_name_teamX_uuidB", "gpt-4o"],
|
||||
router,
|
||||
{},
|
||||
)
|
||||
# public id for the client; an internal routing key for the metadata lookup
|
||||
assert entries[0][0] == "tushar-gpt-4.1"
|
||||
assert entries[0][1].startswith("model_name_teamX_uuid")
|
||||
assert entries[1] == ("gpt-4o", "gpt-4o")
|
||||
assert len(entries) == 2
|
||||
|
||||
|
||||
def test_listing_entries_lookup_id_never_crosses_team_boundary():
|
||||
"""Regression: when two teams share a team_public_model_name, the lookup id for
|
||||
the shared public name must stay within the caller's accessible model_names and
|
||||
never resolve to the other team's internal routing key (which would leak that
|
||||
team's fallback metadata under include_metadata=true)."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "shared-name",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_teamY_uuidB", # different team, same public name
|
||||
"model_info": {
|
||||
"team_id": "teamY",
|
||||
"team_public_model_name": "shared-name",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# caller can only access teamX's internal key
|
||||
entries = TeamModelNameTranslator.listing_entries(
|
||||
["model_name_teamX_uuidA"], router, {}
|
||||
)
|
||||
|
||||
assert entries == [("shared-name", "model_name_teamX_uuidA")]
|
||||
|
||||
|
||||
def test_listing_entries_global_wins_when_team_alias_collides_with_global():
|
||||
"""Regression: when an accessible global model shares its name with a team
|
||||
deployment's `team_public_model_name`, the listing must keep the global
|
||||
entry rather than overwriting its lookup id with the colliding team's
|
||||
internal routing key (which would surface the team's metadata under the
|
||||
global id)."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "gpt-4o",
|
||||
},
|
||||
},
|
||||
{"model_name": "gpt-4o", "model_info": {"db_model": False}},
|
||||
]
|
||||
|
||||
entries = TeamModelNameTranslator.listing_entries(
|
||||
["gpt-4o", "model_name_teamX_uuidA"], router, {}
|
||||
)
|
||||
|
||||
assert entries == [("gpt-4o", "gpt-4o")]
|
||||
|
||||
|
||||
def test_listing_and_resolve_agree_on_sibling_internal_key():
|
||||
"""Regression: when two team deployments share a public name, listing and
|
||||
retrieve must pick the same internal routing key, otherwise `/v1/models/{id}`
|
||||
describes a different deployment than what the listing's metadata was built
|
||||
from."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidB",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
},
|
||||
]
|
||||
available = ["model_name_teamX_uuidA", "model_name_teamX_uuidB"]
|
||||
|
||||
[(_, listing_lookup)] = TeamModelNameTranslator.listing_entries(
|
||||
available, router, {}
|
||||
)
|
||||
resolve_lookup = TeamModelNameTranslator.resolve_public_name(
|
||||
model_id="tushar-gpt-4.1",
|
||||
available_models=available,
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
)
|
||||
|
||||
assert listing_lookup == resolve_lookup
|
||||
|
||||
|
||||
def test_listing_entries_skips_empty_team_public_model_name():
|
||||
"""Regression: a misconfigured row with `team_public_model_name: ""` must not
|
||||
produce a listing entry with an empty `id`; the internal routing key should
|
||||
pass through unchanged, matching `/v1/model/info`'s falsy-check behavior."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
entries = TeamModelNameTranslator.listing_entries(
|
||||
["model_name_teamX_uuidA"], router, {}
|
||||
)
|
||||
|
||||
assert entries == [("model_name_teamX_uuidA", "model_name_teamX_uuidA")]
|
||||
|
||||
|
||||
def test_listing_entries_passthrough_when_disabled():
|
||||
"""Legacy flag / no router -> response id equals lookup id (no translation)."""
|
||||
assert TeamModelNameTranslator.listing_entries(["a", "b"], None, {}) == [
|
||||
("a", "a"),
|
||||
("b", "b"),
|
||||
]
|
||||
|
||||
|
||||
def test_translate_team_model_names_for_listing_leaves_unmapped_names():
|
||||
"""Names with no team mapping (globals, access-group keys) pass through."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{"model_name": "gpt-4o", "model_info": {"db_model": False}}
|
||||
]
|
||||
|
||||
assert TeamModelNameTranslator.translate_listing(
|
||||
["gpt-4o", "beta-group"], router, {}
|
||||
) == ["gpt-4o", "beta-group"]
|
||||
|
||||
|
||||
def test_translate_team_model_names_for_listing_none_router():
|
||||
"""No router -> return the input list unchanged."""
|
||||
assert TeamModelNameTranslator.translate_listing(["a", "b"], None, {}) == ["a", "b"]
|
||||
|
||||
|
||||
def test_translate_team_model_names_for_listing_respects_legacy_flag():
|
||||
"""Operators can keep returning the legacy internal routing key."""
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = [
|
||||
{
|
||||
"model_name": "model_name_teamX_uuidA",
|
||||
"model_info": {
|
||||
"team_id": "teamX",
|
||||
"team_public_model_name": "tushar-gpt-4.1",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
assert TeamModelNameTranslator.translate_listing(
|
||||
["model_name_teamX_uuidA"], router, {"use_team_public_model_name": False}
|
||||
) == ["model_name_teamX_uuidA"]
|
||||
|
||||
|
||||
def _public_named_router(*team_rows: dict) -> MagicMock:
|
||||
router = MagicMock()
|
||||
router.get_model_list.return_value = list(team_rows)
|
||||
return router
|
||||
|
||||
|
||||
def test_resolve_public_name_to_internal_routing_key():
|
||||
"""A public team name resolves back to the internal routing key the router
|
||||
indexes by, so `GET /v1/models/{public_name}` can find the deployment."""
|
||||
router = _public_named_router(_team_row())
|
||||
|
||||
assert (
|
||||
TeamModelNameTranslator.resolve_public_name(
|
||||
model_id="team-claude-sonnet",
|
||||
available_models=["model_name_team-abc-123_4a6b8"],
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
)
|
||||
== "model_name_team-abc-123_4a6b8"
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_public_name_is_access_scoped_across_teams():
|
||||
"""Two teams can publish the SAME public name. A caller's query must resolve
|
||||
to the internal key they can actually access, never another team's."""
|
||||
# both rows share public name "team-claude-sonnet"
|
||||
router = _public_named_router(_team_row(), _other_team_row())
|
||||
|
||||
# caller only has access to their own team's internal key
|
||||
resolved = TeamModelNameTranslator.resolve_public_name(
|
||||
model_id="team-claude-sonnet",
|
||||
available_models=["model_name_team-abc-123_4a6b8"],
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
)
|
||||
assert resolved == "model_name_team-abc-123_4a6b8"
|
||||
assert resolved != "model_name_team-other_9f2c1"
|
||||
|
||||
|
||||
def test_resolve_public_name_unmapped_passes_through():
|
||||
"""A public name with no accessible internal mapping is returned unchanged so
|
||||
the caller hits the normal 404/access path; internal names pass through too."""
|
||||
router = _public_named_router(_team_row())
|
||||
|
||||
# not accessible -> unchanged (downstream validate_model_access will 404)
|
||||
assert (
|
||||
TeamModelNameTranslator.resolve_public_name(
|
||||
model_id="team-claude-sonnet",
|
||||
available_models=[],
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
)
|
||||
== "team-claude-sonnet"
|
||||
)
|
||||
# already an internal routing key -> unchanged
|
||||
assert (
|
||||
TeamModelNameTranslator.resolve_public_name(
|
||||
model_id="model_name_team-abc-123_4a6b8",
|
||||
available_models=["model_name_team-abc-123_4a6b8"],
|
||||
llm_router=router,
|
||||
general_settings={},
|
||||
)
|
||||
== "model_name_team-abc-123_4a6b8"
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_public_name_respects_legacy_flag():
|
||||
"""With the legacy flag set, no public-name resolution happens."""
|
||||
router = _public_named_router(_team_row())
|
||||
|
||||
assert (
|
||||
TeamModelNameTranslator.resolve_public_name(
|
||||
model_id="team-claude-sonnet",
|
||||
available_models=["model_name_team-abc-123_4a6b8"],
|
||||
llm_router=router,
|
||||
general_settings={"use_team_public_model_name": False},
|
||||
)
|
||||
== "team-claude-sonnet"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve_model_by_public_name_returns_200(monkeypatch):
|
||||
"""Regression: `GET /v1/models/{public_name}` must NOT 404. The listing
|
||||
advertises the public team name, so retrieve must accept the same name,
|
||||
resolve it to the internal routing key for lookup, and echo the public name
|
||||
back as the model id."""
|
||||
import litellm
|
||||
import litellm.proxy.utils as proxy_utils
|
||||
|
||||
team_row = _team_row()
|
||||
router = _public_named_router(team_row)
|
||||
deployment = MagicMock()
|
||||
deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing"
|
||||
router.get_deployment_by_model_group_name.return_value = deployment
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
monkeypatch.setattr(
|
||||
proxy_utils,
|
||||
"get_available_models_for_user",
|
||||
AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm, "get_llm_provider", lambda model: (model, "openai", None, None)
|
||||
)
|
||||
|
||||
key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[])
|
||||
resp = await ps.model_info(model_id="team-claude-sonnet", user_api_key_dict=key)
|
||||
|
||||
assert resp["id"] == "team-claude-sonnet"
|
||||
# lookup happened by the internal routing key, not the public name
|
||||
router.get_deployment_by_model_group_name.assert_called_once_with(
|
||||
"model_name_team-abc-123_4a6b8"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve_model_by_internal_name_returns_public_id(monkeypatch):
|
||||
"""Regression: retrieving by the internal routing key must echo the SAME
|
||||
public id `/v1/models` advertises for that deployment, not the path. Otherwise
|
||||
a client iterating the listing's id and then retrieving each one would observe
|
||||
a different id depending on which alias they queried by."""
|
||||
import litellm
|
||||
import litellm.proxy.utils as proxy_utils
|
||||
|
||||
router = _public_named_router(_team_row())
|
||||
deployment = MagicMock()
|
||||
deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing"
|
||||
router.get_deployment_by_model_group_name.return_value = deployment
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
monkeypatch.setattr(
|
||||
proxy_utils,
|
||||
"get_available_models_for_user",
|
||||
AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm, "get_llm_provider", lambda model: (model, "openai", None, None)
|
||||
)
|
||||
|
||||
key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[])
|
||||
resp = await ps.model_info(
|
||||
model_id="model_name_team-abc-123_4a6b8", user_api_key_dict=key
|
||||
)
|
||||
|
||||
assert resp["id"] == "team-claude-sonnet"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve_model_by_internal_name_keeps_internal_id_when_flag_disabled(
|
||||
monkeypatch,
|
||||
):
|
||||
"""With `use_team_public_model_name=false`, retrieve must keep the internal
|
||||
routing key as the response id, mirroring `/v1/models`' legacy output."""
|
||||
import litellm
|
||||
import litellm.proxy.utils as proxy_utils
|
||||
|
||||
router = _public_named_router(_team_row())
|
||||
deployment = MagicMock()
|
||||
deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing"
|
||||
router.get_deployment_by_model_group_name.return_value = deployment
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "general_settings", {"use_team_public_model_name": False})
|
||||
monkeypatch.setattr(
|
||||
proxy_utils,
|
||||
"get_available_models_for_user",
|
||||
AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm, "get_llm_provider", lambda model: (model, "openai", None, None)
|
||||
)
|
||||
|
||||
key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[])
|
||||
resp = await ps.model_info(
|
||||
model_id="model_name_team-abc-123_4a6b8", user_api_key_dict=key
|
||||
)
|
||||
|
||||
assert resp["id"] == "model_name_team-abc-123_4a6b8"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retrieve_model_by_inaccessible_public_name_404s(monkeypatch):
|
||||
"""A caller without access to a team model still gets 404 when retrieving by
|
||||
its public name; resolution never crosses the access boundary."""
|
||||
import litellm
|
||||
import litellm.proxy.utils as proxy_utils
|
||||
|
||||
router = _public_named_router(_team_row())
|
||||
deployment = MagicMock()
|
||||
deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing"
|
||||
router.get_deployment_by_model_group_name.return_value = deployment
|
||||
|
||||
monkeypatch.setattr(ps, "llm_router", router)
|
||||
monkeypatch.setattr(ps, "general_settings", {})
|
||||
monkeypatch.setattr(
|
||||
proxy_utils,
|
||||
"get_available_models_for_user",
|
||||
AsyncMock(return_value=[]), # caller has no access
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm, "get_llm_provider", lambda model: (model, "openai", None, None)
|
||||
)
|
||||
|
||||
key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[])
|
||||
with pytest.raises(ps.HTTPException) as exc_info:
|
||||
await ps.model_info(model_id="team-claude-sonnet", user_api_key_dict=key)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
router.get_deployment_by_model_group_name.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -359,6 +359,7 @@ ignored_keys = [
|
|||
"metadata.additional_usage_values.cache_read_input_tokens",
|
||||
"metadata.additional_usage_values.inference_geo",
|
||||
"metadata.additional_usage_values.speed",
|
||||
"metadata.additional_usage_values.service_tier",
|
||||
"metadata.litellm_overhead_time_ms",
|
||||
"metadata.cost_breakdown",
|
||||
"metadata.user_api_key",
|
||||
|
|
|
|||
92
tests/test_litellm/proxy/test_model_list_healthy_only.py
Normal file
92
tests/test_litellm/proxy/test_model_list_healthy_only.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
"""
|
||||
Tests for the opt-in `healthy_only` filter on GET /v1/models (`model_list`).
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched_model_list(monkeypatch):
|
||||
"""Stub router + utility helpers used by `model_list`."""
|
||||
from litellm.proxy import utils as proxy_utils
|
||||
|
||||
router = MagicMock()
|
||||
router.get_fully_blocked_model_names = MagicMock(return_value=set())
|
||||
router.async_get_fully_unhealthy_model_names = AsyncMock(
|
||||
return_value={"claude-sonnet"}
|
||||
)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "user_model", None)
|
||||
|
||||
async def _fake_get_available_models_for_user(**kwargs):
|
||||
return ["gpt-4", "claude-sonnet"]
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_utils,
|
||||
"get_available_models_for_user",
|
||||
_fake_get_available_models_for_user,
|
||||
)
|
||||
|
||||
def _fake_create_model_info_response(model_id, provider="openai", **kwargs):
|
||||
return {"id": model_id, "object": "model", "created": 0, "owned_by": provider}
|
||||
|
||||
monkeypatch.setattr(
|
||||
proxy_utils, "create_model_info_response", _fake_create_model_info_response
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_list_healthy_only_hides_fully_unhealthy_models(
|
||||
patched_model_list,
|
||||
):
|
||||
response = await proxy_server.model_list(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
healthy_only=True,
|
||||
)
|
||||
assert [m["id"] for m in response["data"]] == ["gpt-4"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_list_default_keeps_unhealthy_models(patched_model_list):
|
||||
response = await proxy_server.model_list(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
)
|
||||
assert [m["id"] for m in response["data"]] == ["gpt-4", "claude-sonnet"]
|
||||
patched_model_list.async_get_fully_unhealthy_model_names.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_list_healthy_only_applies_to_scope_expand(
|
||||
patched_model_list, monkeypatch
|
||||
):
|
||||
from litellm.proxy.auth import model_checks
|
||||
from litellm.proxy.management_endpoints import common_utils
|
||||
|
||||
async def _fake_admin(**kwargs):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(common_utils, "_user_has_admin_privileges", _fake_admin)
|
||||
monkeypatch.setattr(
|
||||
model_checks,
|
||||
"get_complete_model_list",
|
||||
lambda **kwargs: ["gpt-4", "claude-sonnet"],
|
||||
)
|
||||
patched_model_list.get_model_names = MagicMock(
|
||||
return_value=["gpt-4", "claude-sonnet"]
|
||||
)
|
||||
patched_model_list.get_model_access_groups = MagicMock(return_value={})
|
||||
|
||||
response = await proxy_server.model_list(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
|
||||
scope="expand",
|
||||
healthy_only=True,
|
||||
)
|
||||
assert [m["id"] for m in response["data"]] == ["gpt-4"]
|
||||
|
|
@ -1948,6 +1948,221 @@ def test_completion_cost_service_tier_for_bedrock():
|
|||
assert priority_cost > default_cost > flex_cost > 0
|
||||
|
||||
|
||||
def test_completion_cost_service_tier_for_anthropic():
|
||||
"""
|
||||
Anthropic priority-tier requests must be priced at the priority rate.
|
||||
|
||||
Regression for LIT-3771: the Anthropic cost route dropped ``service_tier``,
|
||||
so priority requests (whose tier is reported on the response usage) were
|
||||
always billed at the standard rate. The tier is captured by the
|
||||
transformation and must flow through to ``generic_cost_per_token``.
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model = "claude-test-service-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_priority": 6e-6,
|
||||
"output_cost_per_token_priority": 30e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def _cost_for_tier(service_tier):
|
||||
usage = AnthropicConfig().calculate_usage(
|
||||
usage_object={
|
||||
"input_tokens": 1000,
|
||||
"output_tokens": 500,
|
||||
"service_tier": service_tier,
|
||||
},
|
||||
reasoning_content=None,
|
||||
)
|
||||
response = ModelResponse(usage=usage, model=model)
|
||||
return completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
standard_cost = _cost_for_tier("standard")
|
||||
priority_cost = _cost_for_tier("priority")
|
||||
|
||||
expected_standard = 1000 * 3e-6 + 500 * 15e-6
|
||||
assert standard_cost == pytest.approx(expected_standard)
|
||||
# priority rates are exactly 2x standard for both input and output
|
||||
assert priority_cost == pytest.approx(2 * standard_cost)
|
||||
|
||||
|
||||
def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate():
|
||||
"""
|
||||
Proxy billing path regression for LIT-3771.
|
||||
|
||||
Priority is opted into with ``service_tier="auto"``; Anthropic then serves
|
||||
"priority" and reports it on the response usage. The proxy forwards the
|
||||
request-level "auto" into ``completion_cost`` (via ``_response_cost_calculator``),
|
||||
and that preference must not shadow the served tier, otherwise priority
|
||||
requests are silently billed at the standard rate.
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model = "claude-test-auto-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_priority": 6e-6,
|
||||
"output_cost_per_token_priority": 30e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
usage = AnthropicConfig().calculate_usage(
|
||||
usage_object={
|
||||
"input_tokens": 1000,
|
||||
"output_tokens": 500,
|
||||
"service_tier": "priority",
|
||||
},
|
||||
reasoning_content=None,
|
||||
)
|
||||
response = ModelResponse(usage=usage, model=model)
|
||||
|
||||
cost = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="anthropic",
|
||||
service_tier="auto",
|
||||
optional_params={"service_tier": "auto"},
|
||||
)
|
||||
|
||||
expected_priority = 1000 * 6e-6 + 500 * 30e-6
|
||||
assert cost == pytest.approx(expected_priority)
|
||||
|
||||
|
||||
def test_completion_cost_non_string_service_tier_defers_to_served_tier():
|
||||
"""
|
||||
Regression: a non-string request-level ``service_tier`` (reachable via
|
||||
``allowed_openai_params``/``drop_params``) must not crash cost tracking.
|
||||
|
||||
Before the fix, ``completion_cost`` called ``service_tier.lower()`` on the
|
||||
request-level value, so a dict raised ``AttributeError``. ``_response_cost_calculator``
|
||||
swallowed it and reported ``response_cost=None``, silently dropping the cost.
|
||||
The non-string preference must be ignored so pricing defers to the tier the
|
||||
provider actually served on the response usage.
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model = "claude-test-non-string-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"input_cost_per_token_priority": 6e-6,
|
||||
"output_cost_per_token_priority": 30e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
usage = AnthropicConfig().calculate_usage(
|
||||
usage_object={
|
||||
"input_tokens": 1000,
|
||||
"output_tokens": 500,
|
||||
"service_tier": "priority",
|
||||
},
|
||||
reasoning_content=None,
|
||||
)
|
||||
response = ModelResponse(usage=usage, model=model)
|
||||
|
||||
cost = completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="anthropic",
|
||||
optional_params={"service_tier": {"name": "auto"}},
|
||||
)
|
||||
|
||||
expected_priority = 1000 * 6e-6 + 500 * 30e-6
|
||||
assert cost == pytest.approx(expected_priority)
|
||||
|
||||
|
||||
def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier():
|
||||
"""
|
||||
Regression for the cache/tier interaction in the Anthropic geo/speed path.
|
||||
|
||||
When a request is served at "priority" and also carries a geo/speed
|
||||
multiplier (here ``speed="fast"``), the cache portion is held out of the
|
||||
multiplier so it is not scaled. That held-out cache cost must use the
|
||||
served tier's cache rate; pricing it at the standard rate while the cache
|
||||
embedded in ``prompt_cost`` is priced at the priority rate leaves a
|
||||
``(cache_priority - cache_standard)(multiplier - 1)`` billing error.
|
||||
"""
|
||||
from litellm.llms.anthropic.cost_calculation import (
|
||||
cost_per_token as anthropic_cost_per_token,
|
||||
)
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
|
||||
model = "claude-test-priority-cache-fast-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 15e-6,
|
||||
"cache_read_input_token_cost": 0.3e-6,
|
||||
"input_cost_per_token_priority": 6e-6,
|
||||
"output_cost_per_token_priority": 30e-6,
|
||||
"cache_read_input_token_cost_priority": 0.6e-6,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_tokens": 8192,
|
||||
"provider_specific_entry": {"fast": 2.0},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=1000,
|
||||
completion_tokens=500,
|
||||
total_tokens=1500,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200),
|
||||
)
|
||||
usage.speed = "fast"
|
||||
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(
|
||||
model=model, usage=usage, service_tier="priority"
|
||||
)
|
||||
|
||||
# non-cache input priced at the priority rate and scaled by the fast
|
||||
# multiplier; the 200 cache-hit tokens priced at the priority cache rate
|
||||
# and held out of the multiplier
|
||||
expected_prompt = (1000 - 200) * 6e-6 * 2 + 200 * 0.6e-6
|
||||
expected_completion = 500 * 30e-6 * 2
|
||||
assert prompt_cost == pytest.approx(expected_prompt)
|
||||
assert completion_cost == pytest.approx(expected_completion)
|
||||
|
||||
|
||||
def test_gemini_cache_tokens_details_no_negative_values():
|
||||
"""
|
||||
Test for Issue #18750: Negative text_tokens with Gemini caching
|
||||
|
|
|
|||
|
|
@ -4218,6 +4218,82 @@ def test_get_fully_blocked_model_names_treats_missing_key_as_unblocked():
|
|||
assert router.get_fully_blocked_model_names() == set()
|
||||
|
||||
|
||||
def _seed_unhealthy_states(router, unhealthy_ids, timestamp=None):
|
||||
import time
|
||||
|
||||
ts = timestamp if timestamp is not None else time.time()
|
||||
router.health_state_cache.set_deployment_health_states(
|
||||
{
|
||||
uid: {"is_healthy": False, "timestamp": ts, "reason": "test_unhealthy"}
|
||||
for uid in unhealthy_ids
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_fully_unhealthy_model_names_marks_name_when_all_unhealthy():
|
||||
router = _router_with_two_deployments([False, False])
|
||||
_seed_unhealthy_states(router, {"dep-0", "dep-1"})
|
||||
assert await router.async_get_fully_unhealthy_model_names() == {"gpt-4o"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_fully_unhealthy_model_names_keeps_name_when_partial():
|
||||
router = _router_with_two_deployments([False, False])
|
||||
_seed_unhealthy_states(router, {"dep-0"})
|
||||
assert await router.async_get_fully_unhealthy_model_names() == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_fully_unhealthy_model_names_empty_without_health_state():
|
||||
router = _router_with_two_deployments([False, False])
|
||||
assert await router.async_get_fully_unhealthy_model_names() == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_fully_unhealthy_model_names_ignores_stale_state():
|
||||
import time
|
||||
|
||||
router = _router_with_two_deployments([False, False])
|
||||
stale_ts = time.time() - (router.health_state_cache.staleness_threshold + 10)
|
||||
_seed_unhealthy_states(router, {"dep-0", "dep-1"}, timestamp=stale_ts)
|
||||
assert await router.async_get_fully_unhealthy_model_names() == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_fully_unhealthy_model_names_includes_team_alias():
|
||||
import litellm
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o",
|
||||
"litellm_params": {"model": "openai/gpt-4o"},
|
||||
"model_info": {
|
||||
"id": "dep-0",
|
||||
"team_id": "team-1",
|
||||
"team_public_model_name": "team-gpt",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
_seed_unhealthy_states(router, {"dep-0"})
|
||||
assert await router.async_get_fully_unhealthy_model_names() == {
|
||||
"gpt-4o",
|
||||
"team-gpt",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_fully_unhealthy_model_names_noop_with_allowed_fails_policy():
|
||||
from litellm.types.router import AllowedFailsPolicy
|
||||
|
||||
router = _router_with_two_deployments([False, False])
|
||||
router.allowed_fails_policy = AllowedFailsPolicy(BadRequestErrorAllowedFails=1)
|
||||
_seed_unhealthy_states(router, {"dep-0", "dep-1"})
|
||||
assert await router.async_get_fully_unhealthy_model_names() == set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_healthy_deployments_skips_blocked_deployment():
|
||||
router = _router_with_two_deployments([True, False])
|
||||
|
|
|
|||
18
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
18
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -6279,6 +6279,10 @@ export interface paths {
|
|||
*
|
||||
* Follows OpenAI API specification for individual model retrieval.
|
||||
* https://platform.openai.com/docs/api-reference/models/retrieve
|
||||
*
|
||||
* Query parameters mirror `/v1/models` so the same caller context (team
|
||||
* scoping, health filtering, paused deployments) drives both endpoints; the
|
||||
* listing's public id must resolve to the same internal deployment here.
|
||||
*/
|
||||
get: operations["model_info_models__model_id__get"];
|
||||
put?: never;
|
||||
|
|
@ -14472,6 +14476,10 @@ export interface paths {
|
|||
*
|
||||
* Follows OpenAI API specification for individual model retrieval.
|
||||
* https://platform.openai.com/docs/api-reference/models/retrieve
|
||||
*
|
||||
* Query parameters mirror `/v1/models` so the same caller context (team
|
||||
* scoping, health filtering, paused deployments) drives both endpoints; the
|
||||
* listing's public id must resolve to the same internal deployment here.
|
||||
*/
|
||||
get: operations["model_info_v1_models__model_id__get"];
|
||||
put?: never;
|
||||
|
|
@ -38145,7 +38153,10 @@ export interface operations {
|
|||
};
|
||||
model_info_models__model_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
query?: {
|
||||
team_id?: string | null;
|
||||
healthy_only?: boolean | null;
|
||||
};
|
||||
header?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
|
|
@ -48211,7 +48222,10 @@ export interface operations {
|
|||
};
|
||||
model_info_v1_models__model_id__get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
query?: {
|
||||
team_id?: string | null;
|
||||
healthy_only?: boolean | null;
|
||||
};
|
||||
header?: never;
|
||||
path: {
|
||||
model_id: string;
|
||||
|
|
|
|||
4
uv.lock
generated
4
uv.lock
generated
|
|
@ -9,7 +9,7 @@ resolution-markers = [
|
|||
]
|
||||
|
||||
[options]
|
||||
exclude-newer = "2026-06-13T01:42:46.429412Z"
|
||||
exclude-newer = "2026-06-15T02:02:32.823508Z"
|
||||
exclude-newer-span = "P3D"
|
||||
|
||||
[manifest]
|
||||
|
|
@ -3280,7 +3280,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "litellm"
|
||||
version = "1.89.1"
|
||||
version = "1.89.2"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue