From ca7d360d87ff2c3638117ddb894335822d82ebd9 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Sat, 13 Jun 2026 11:42:43 -0700 Subject: [PATCH 1/9] fix(otel): record full error message on standard exception event in otel v2 (#30380) The v2 span engine only stamped error.type and stuffed the message into the span status description; it never recorded the standard OTel exception event. Backends that dynamic-map unknown string fields (e.g. Elasticsearch) index the message as a keyword capped at ignore_above:1024, truncating it. Emit the full message under the recognized exception.message semconv field via a span event so it is mapped as full text instead. Co-authored-by: Claude (cherry picked from commit 3b84150137d06d0c52d9db10f42bc7c1d540f1e1) --- litellm/integrations/otel/emitter.py | 16 +++- litellm/integrations/otel/model/semconv.py | 15 ++++ .../otel/test_otel_v2_components.py | 84 +++++++++++++++++++ 3 files changed, 111 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index 7fb7be7ab84..6feaf2734e9 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -17,7 +17,7 @@ from litellm.integrations.otel.model.payloads import ( ServiceSpanData, ) from litellm.integrations.otel.plumbing.providers import to_otel_span_kind -from litellm.integrations.otel.model.semconv import Error +from litellm.integrations.otel.model.semconv import Error, ExceptionEvent from litellm.integrations.otel.model.spans import ( SPAN_REGISTRY, SpanRole, @@ -179,9 +179,17 @@ class SpanEmitter: else None ) if error and (error.error_type or error.message): - span.set_attribute(Error.TYPE, error.error_type or "error") - span.set_status( - Status(StatusCode.ERROR, error.message or error.error_type or "error") + error_type = error.error_type or "error" + message = error.message or error.error_type or "error" + span.set_attribute(Error.TYPE, error_type) + span.set_status(Status(StatusCode.ERROR, message)) + # Carry the full message on the standard ``exception`` event so backends + # map it as full text under ``exception.message``. Setting it as a bare + # string attribute instead lets backends like Elasticsearch dynamic-map + # it to a ``keyword`` capped at 1024 chars, truncating the message. + span.add_event( + ExceptionEvent.NAME, + {ExceptionEvent.TYPE: error_type, ExceptionEvent.MESSAGE: message}, ) # On success leave the status UNSET (the semconv default) rather than # forcing OK — that matches the FastAPI server span and avoids implying a diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 7df07f30a01..bb93a357516 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -146,6 +146,21 @@ class Error: TYPE: Final = "error.type" +class ExceptionEvent: + """OTel exception-event name and attribute keys (semconv ``exception.*``). + + The full error message rides ``exception.message`` on a span event rather than + a custom string attribute. Backends recognise these semantic-convention names + and map them as full text; an unrecognised key (e.g. ``error_message``) falls + into the default dynamic template, which truncates strings to a 1024-char + ``keyword``. + """ + + NAME: Final = "exception" + TYPE: Final = "exception.type" + MESSAGE: Final = "exception.message" + + class Server: ADDRESS: Final = "server.address" PORT: Final = "server.port" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py index 86d84bd8100..c3fa3f424b5 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py @@ -410,6 +410,90 @@ def test_emitter_without_call_id_is_not_deduped(): assert len(exporter.get_finished_spans()) == 2 +def _emit_error_span(message, error_type="litellm.APIError"): + from litellm.integrations.otel.emitter import SpanEmitter + + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg) + data = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model=None, + response_id=None, + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=(), + error=SpanError(error_type=error_type, message=message), + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + engine.emit(SpanRole.LLM_CALL, data) + (span,) = exporter.get_finished_spans() + return span + + +def _exception_event(span): + from litellm.integrations.otel.model.semconv import ExceptionEvent + + events = [e for e in span.events if e.name == ExceptionEvent.NAME] + assert len(events) == 1, "expected exactly one exception event" + return events[0] + + +def test_error_message_recorded_as_full_exception_event_untruncated(): + """Regression for the Elasticsearch keyword/ignore_above:1024 truncation. + + A long error message must survive intact on the standard ``exception`` + event under ``exception.message`` — not get dropped onto a bare string + attribute that backends dynamic-map to a 1024-char ``keyword``. The SDK + must not truncate it either, so a 5000-char message stays 5000 chars. + """ + from litellm.integrations.otel.model.semconv import Error, ExceptionEvent + + long_message = "boom: " + "x" * 5000 + span = _emit_error_span(long_message, error_type="litellm.APIError") + + event = _exception_event(span) + assert event.attributes[ExceptionEvent.MESSAGE] == long_message + assert len(event.attributes[ExceptionEvent.MESSAGE]) == len(long_message) > 1024 + assert event.attributes[ExceptionEvent.TYPE] == "litellm.APIError" + + # error.type stays a low-cardinality attribute; the message does NOT become a + # bare string attribute (which is what got truncated). + assert span.attributes[Error.TYPE] == "litellm.APIError" + assert ExceptionEvent.MESSAGE not in span.attributes + assert span.status.description == long_message + + +def test_success_span_records_no_exception_event(): + from litellm.integrations.otel.emitter import SpanEmitter + from litellm.integrations.otel.model.semconv import ExceptionEvent + + cfg = OpenTelemetryV2Config(exporter="in_memory") + provider, exporter = providers.in_memory_provider(cfg) + engine = SpanEmitter(providers.get_tracer(provider, "t"), cfg) + data = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model="gpt-4o", + response_id="resp-1", + request_params=LLMRequestParams(), + usage=LLMUsage(), + finish_reasons=("stop",), + error=None, + response_cost=None, + server=None, + identity=RequestIdentity(call_id=None), + ) + engine.emit(SpanRole.LLM_CALL, data) + (span,) = exporter.get_finished_spans() + assert all(e.name != ExceptionEvent.NAME for e in span.events) + + # --- service taxonomy: which calls become spans, and of what kind ----------- # From f8831f37205899f76847ab50c0117ae5bd98b5b7 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Tue, 16 Jun 2026 12:13:31 -0700 Subject: [PATCH 2/9] fix(proxy): allow internal roles to access vector store CRUD routes (#30503) Add bare /v1/vector_stores/{vector_store_id} to openai_routes so retrieve, update, and delete classify as LLM API routes for internal user and internal viewer roles. Co-authored-by: Cursor (cherry picked from commit 902122a06bbba991ccadac73171af1aadf899696) --- litellm/proxy/_types.py | 2 + .../proxy/auth/test_route_checks.py | 59 +++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index c9fe3a0c88e..ad161457326 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -376,6 +376,8 @@ class LiteLLMRoutes(enum.Enum): # vector stores "/vector_stores", "/v1/vector_stores", + "/vector_stores/{vector_store_id}", + "/v1/vector_stores/{vector_store_id}", "/vector_stores/{vector_store_id}/search", "/v1/vector_stores/{vector_store_id}/search", "/vector_stores/{vector_store_id}/files", diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index 63b61954cf6..07b04961205 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -1400,6 +1400,65 @@ def test_rag_routes_accessible_to_internal_user_viewer(): ) +@pytest.mark.parametrize( + "route", + [ + "/vector_stores/vs_123", + "/v1/vector_stores/vs_123", + "/vector_stores/vs_123/search", + "/v1/vector_stores/vs_123/search", + "/vector_stores/vs_123/files", + "/v1/vector_stores/vs_123/files", + ], +) +def test_vector_store_routes_are_llm_api_routes(route): + """Retrieve/update/delete on a single vector store must classify as LLM API routes. + + Regression for the missing bare `/v1/vector_stores/{vector_store_id}` entry in + `openai_routes` that left retrieve/update/delete blocked for internal roles + while `/search` and `/files` sub-routes worked. + """ + + assert RouteChecks.is_llm_api_route(route) is True + + +@pytest.mark.parametrize( + "user_role", + [ + LitellmUserRoles.INTERNAL_USER.value, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value, + ], +) +@pytest.mark.parametrize( + "method, route", + [ + ("GET", "/v1/vector_stores/vs_123"), + ("POST", "/v1/vector_stores/vs_123"), + ("DELETE", "/v1/vector_stores/vs_123"), + ], +) +def test_vector_store_crud_accessible_to_internal_roles(user_role, method, route): + """Internal user and internal viewer must reach vector store retrieve/update/delete. + + Object-level access is still gated by `assert_user_can_access_vector_store`; + this only verifies the route gate no longer 403s these roles. + """ + + valid_token = UserAPIKeyAuth(user_id="test_user", user_role=user_role) + request = MagicMock(spec=Request) + request.method = method + request.query_params = {} + + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="test_user", user_role=user_role), + _user_role=user_role, + route=route, + request=request, + valid_token=valid_token, + request_data={}, + ) + + def test_videos_route_accessible_to_internal_users(): """ Test that internal users can access the videos routes. From 0c40c1e5729d77034d11cdfba393b827b9515611 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 17 Jun 2026 06:33:51 -0700 Subject: [PATCH 3/9] fix(anthropic): price and surface response service_tier in cost tracking (#30558) (cherry picked from commit b638bc2248e7ed600b7bb951146e050ca12722bb) --- litellm/cost_calculator.py | 11 +- litellm/llms/anthropic/chat/transformation.py | 5 + litellm/llms/anthropic/cost_calculation.py | 23 ++- litellm/types/utils.py | 1 + .../test_anthropic_chat_transformation.py | 33 ++++ .../test_spend_management_endpoints.py | 1 + tests/test_litellm/test_cost_calculator.py | 163 ++++++++++++++++++ 7 files changed, 231 insertions(+), 6 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 88029615ba8..6c11c8d8a06 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -94,6 +94,7 @@ from litellm.types.utils import ( LlmProviders, LlmProvidersSet, ModelInfo, + ServiceTier, StandardBuiltInToolsParams, TranscriptionUsageDurationObject, TranscriptionUsageTokensObject, @@ -614,7 +615,9 @@ def cost_per_token( # noqa: PLR0915 service_tier=service_tier, ) elif custom_llm_provider == "anthropic": - return anthropic_cost_per_token(model=model, usage=usage_block) + return anthropic_cost_per_token( + model=model, usage=usage_block, service_tier=service_tier + ) elif custom_llm_provider == "bedrock": return bedrock_cost_per_token( model=model, usage=usage_block, service_tier=service_tier @@ -1224,6 +1227,12 @@ def completion_cost( # noqa: PLR0915 if service_tier is None and optional_params is not None: service_tier = optional_params.get("service_tier") + # "auto" is a routing preference, not a billable tier: the provider picks + # the tier and reports the one actually served on the response/usage, so + # defer to that instead of pricing the request-level "auto" as standard + if service_tier is not None and service_tier.lower() == ServiceTier.AUTO.value: + service_tier = None + # Extract service_tier from completion_response if not provided if service_tier is None and completion_response is not None: if isinstance(completion_response, BaseModel): diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index cc30db0ebad..7b87ed02a69 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2205,6 +2205,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): inference_geo: Optional[str] = None if "inference_geo" in _usage and _usage["inference_geo"] is not None: inference_geo = _usage["inference_geo"] + service_tier = cast( + str | None, + _usage.get("service_tier"), # any-ok: untyped usage dict + ) if ( "cache_creation_input_tokens" in _usage @@ -2298,6 +2302,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ), inference_geo=inference_geo, speed=speed, + service_tier=service_tier, ) return usage diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 6a031498dae..44081ea9e79 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -18,7 +18,9 @@ if TYPE_CHECKING: import litellm -def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: +def _compute_cache_only_cost( + model_info: "ModelInfo", usage: "Usage", service_tier: str | None = None +) -> float: """ Return only the cache-related portion of the prompt cost (cache read + cache write). @@ -36,7 +38,9 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: cache_creation_cost, cache_creation_cost_above_1hr, cache_read_cost, - ) = _get_token_base_cost(model_info=model_info, usage=usage) + ) = _get_token_base_cost( + model_info=model_info, usage=usage, service_tier=service_tier + ) cache_cost = float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost @@ -56,19 +60,26 @@ def _compute_cache_only_cost(model_info: "ModelInfo", usage: "Usage") -> float: return cache_cost -def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: +def cost_per_token( + model: str, usage: "Usage", service_tier: str | None = None +) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. Input: - model: str, the model name without provider prefix - usage: LiteLLM Usage block, containing anthropic caching information + - service_tier: the service tier the request was served at (e.g. "priority"), + read from the Anthropic response usage and used to select tier-specific pricing Returns: Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd """ prompt_cost, completion_cost = generic_cost_per_token( - model=model, usage=usage, custom_llm_provider="anthropic" + model=model, + usage=usage, + custom_llm_provider="anthropic", + service_tier=service_tier, ) # Apply provider_specific_entry multipliers for geo/speed routing @@ -89,7 +100,9 @@ def cost_per_token(model: str, usage: "Usage") -> Tuple[float, float]: multiplier *= provider_specific_entry.get("fast", 1.0) if multiplier != 1.0: - cache_cost = _compute_cache_only_cost(model_info=model_info, usage=usage) + cache_cost = _compute_cache_only_cost( + model_info=model_info, usage=usage, service_tier=service_tier + ) prompt_cost = (prompt_cost - cache_cost) * multiplier + cache_cost completion_cost *= multiplier except Exception: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0200d1a0831..220d1861eb8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3635,6 +3635,7 @@ class SpecialEnums(Enum): class ServiceTier(Enum): """Enum for service tier types used in cost calculations.""" + AUTO = "auto" FLEX = "flex" PRIORITY = "priority" diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index c91c9c3fdf4..ce75ccdccce 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -3702,6 +3702,39 @@ def test_fast_mode_with_inference_geo(): assert abs(completion_cost - base_completion * expected_multiplier) < 1e-10 +def test_calculate_usage_captures_service_tier(): + """ + Anthropic returns the assigned service tier on the response usage object + (e.g. ``"priority"``). It must be surfaced on the Usage object so it is + visible in logs and used to select tier-specific pricing. + """ + config = AnthropicConfig() + + usage_object = { + "input_tokens": 410, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 585, + "service_tier": "priority", + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + assert usage.service_tier == "priority" + + +def test_calculate_usage_service_tier_defaults_to_none(): + """A response without a service tier must not invent one.""" + config = AnthropicConfig() + + usage = config.calculate_usage( + usage_object={"input_tokens": 10, "output_tokens": 5}, + reasoning_content=None, + ) + + assert usage.service_tier is None + + def test_fast_mode_parameter_in_supported_params(): """ Test that 'speed' is in the list of supported OpenAI params. diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index aef91ed3c77..bc8b4474a7d 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -359,6 +359,7 @@ ignored_keys = [ "metadata.additional_usage_values.cache_read_input_tokens", "metadata.additional_usage_values.inference_geo", "metadata.additional_usage_values.speed", + "metadata.additional_usage_values.service_tier", "metadata.litellm_overhead_time_ms", "metadata.cost_breakdown", "metadata.user_api_key", diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 82a4a60bf82..bcc685f0366 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1948,6 +1948,169 @@ def test_completion_cost_service_tier_for_bedrock(): assert priority_cost > default_cost > flex_cost > 0 +def test_completion_cost_service_tier_for_anthropic(): + """ + Anthropic priority-tier requests must be priced at the priority rate. + + Regression for LIT-3771: the Anthropic cost route dropped ``service_tier``, + so priority requests (whose tier is reported on the response usage) were + always billed at the standard rate. The tier is captured by the + transformation and must flow through to ``generic_cost_per_token``. + """ + from litellm import completion_cost + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-service-tier-cost-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + } + } + ) + + def _cost_for_tier(service_tier): + usage = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 1000, + "output_tokens": 500, + "service_tier": service_tier, + }, + reasoning_content=None, + ) + response = ModelResponse(usage=usage, model=model) + return completion_cost( + completion_response=response, + model=model, + custom_llm_provider="anthropic", + ) + + standard_cost = _cost_for_tier("standard") + priority_cost = _cost_for_tier("priority") + + expected_standard = 1000 * 3e-6 + 500 * 15e-6 + assert standard_cost == pytest.approx(expected_standard) + # priority rates are exactly 2x standard for both input and output + assert priority_cost == pytest.approx(2 * standard_cost) + + +def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(): + """ + Proxy billing path regression for LIT-3771. + + Priority is opted into with ``service_tier="auto"``; Anthropic then serves + "priority" and reports it on the response usage. The proxy forwards the + request-level "auto" into ``completion_cost`` (via ``_response_cost_calculator``), + and that preference must not shadow the served tier, otherwise priority + requests are silently billed at the standard rate. + """ + from litellm import completion_cost + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-auto-tier-cost-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + } + } + ) + + usage = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 1000, + "output_tokens": 500, + "service_tier": "priority", + }, + reasoning_content=None, + ) + response = ModelResponse(usage=usage, model=model) + + cost = completion_cost( + completion_response=response, + model=model, + custom_llm_provider="anthropic", + service_tier="auto", + optional_params={"service_tier": "auto"}, + ) + + expected_priority = 1000 * 6e-6 + 500 * 30e-6 + assert cost == pytest.approx(expected_priority) + + +def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(): + """ + Regression for the cache/tier interaction in the Anthropic geo/speed path. + + When a request is served at "priority" and also carries a geo/speed + multiplier (here ``speed="fast"``), the cache portion is held out of the + multiplier so it is not scaled. That held-out cache cost must use the + served tier's cache rate; pricing it at the standard rate while the cache + embedded in ``prompt_cost`` is priced at the priority rate leaves a + ``(cache_priority - cache_standard)(multiplier - 1)`` billing error. + """ + from litellm.llms.anthropic.cost_calculation import ( + cost_per_token as anthropic_cost_per_token, + ) + from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-priority-cache-fast-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "cache_read_input_token_cost": 0.3e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "cache_read_input_token_cost_priority": 0.6e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + "provider_specific_entry": {"fast": 2.0}, + } + } + ) + + usage = Usage( + prompt_tokens=1000, + completion_tokens=500, + total_tokens=1500, + prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=200), + ) + usage.speed = "fast" + + prompt_cost, completion_cost = anthropic_cost_per_token( + model=model, usage=usage, service_tier="priority" + ) + + # non-cache input priced at the priority rate and scaled by the fast + # multiplier; the 200 cache-hit tokens priced at the priority cache rate + # and held out of the multiplier + expected_prompt = (1000 - 200) * 6e-6 * 2 + 200 * 0.6e-6 + expected_completion = 500 * 30e-6 * 2 + assert prompt_cost == pytest.approx(expected_prompt) + assert completion_cost == pytest.approx(expected_completion) + + def test_gemini_cache_tokens_details_no_negative_values(): """ Test for Issue #18750: Negative text_tokens with Gemini caching From 98636c7d6ea4dddf50a9b95a567b55b1ae21c968 Mon Sep 17 00:00:00 2001 From: "T. Kobayashi" <13004314+nix-tkobayashi@users.noreply.github.com> Date: Thu, 11 Jun 2026 21:05:47 +0900 Subject: [PATCH 4/9] feat: add opt-in healthy_only filter to GET /v1/models (#30130) * feat: add opt-in healthy_only filter to GET /v1/models Adds an opt-in `healthy_only=true` query parameter to GET /v1/models and GET /models that hides models whose backing deployments are all marked unhealthy by background health checks. - Add Router.async_get_fully_unhealthy_model_names(), mirroring the semantics of get_fully_blocked_model_names(): a model is hidden only when every backing deployment is unhealthy and the health state is not stale (fail open otherwise). - Reuses the existing DeploymentHealthCache populated by _run_background_health_check(), so no new health state is introduced. - No-op when allowed_fails_policy is set, mirroring _async_filter_health_check_unhealthy_deployments semantics. - team_public_model_name aliases are aggregated alongside model_name. - Hiding is presentation-only; default behavior is unchanged. Fixes #30128 Co-Authored-By: Claude Fable 5 * docs: address Greptile review notes - Note team-alias asymmetry vs get_fully_blocked_model_names - Debug-log when healthy_only is set but no health state is available Co-Authored-By: Claude Fable 5 --------- Co-authored-by: shin-berri Co-authored-by: yuneng-jiang Co-authored-by: Claude Fable 5 (cherry picked from commit 9dd9d2322ac95cf46b21602c6723a704636ea78c) --- litellm/proxy/proxy_server.py | 35 +++++-- litellm/router.py | 57 ++++++++++++ .../proxy/test_model_list_healthy_only.py | 92 +++++++++++++++++++ tests/test_litellm/test_router.py | 76 +++++++++++++++ 4 files changed, 254 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/proxy/test_model_list_healthy_only.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a6413138bd5..95e8994ff32 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -8202,6 +8202,7 @@ async def model_list( include_metadata: Optional[bool] = False, fallback_type: Optional[str] = None, scope: Optional[str] = None, + healthy_only: Optional[bool] = False, ): """ Use `/model/info` - to get detailed model information, example - pricing, mode, etc. @@ -8215,6 +8216,15 @@ async def model_list( - scope: Optional scope parameter. Currently only accepts "expand". When scope=expand is passed, proxy admins, team admins, and org admins will receive all proxy models as if they are a proxy admin. + - healthy_only: When true, hide models whose backing deployments are all marked + unhealthy by background health checks. Requires + `background_health_checks: true` in general_settings; without + health state the listing is returned unfiltered (fail open). + Models expanded from wildcard routes (e.g. `openai/*`) are not + filtered, and nothing is hidden when `allowed_fails_policy` is + configured (cooldown remains the sole exclusion mechanism). + Hiding is presentation-only: a hidden model can still be + called directly. """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj @@ -8248,6 +8258,19 @@ async def model_list( llm_router.get_fully_blocked_model_names() if llm_router is not None else set() ) + # Opt-in: also hide models whose deployments are all unhealthy per background + # health checks. Empty when health state is unavailable or stale (fail open). + unhealthy_names: Set[str] = set() + if healthy_only and llm_router is not None: + unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names() + if not unhealthy_names: + verbose_proxy_logger.debug( + "healthy_only=true but no unhealthy deployment state is available " + "(requires background_health_checks); returning unfiltered model list" + ) + + hidden_names = blocked_names | unhealthy_names + # If scope=expand and user has admin privileges, return all proxy models if should_expand_scope: # Get all proxy models as if user is a proxy admin @@ -8280,9 +8303,9 @@ async def model_list( only_model_access_groups=only_model_access_groups or False, ) - # Hide paused models from the public listing (admins manage them via /model/info) - if blocked_names: - all_models = [m for m in all_models if m not in blocked_names] + # Hide paused/unhealthy models from the public listing + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] # Build response data with all proxy models model_data = [] @@ -8317,9 +8340,9 @@ async def model_list( user_api_key_cache=user_api_key_cache, ) - # Hide paused models from the public listing (admins manage them via /model/info) - if blocked_names: - all_models = [m for m in all_models if m not in blocked_names] + # Hide paused/unhealthy models from the public listing + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] # Build response data model_data = [] diff --git a/litellm/router.py b/litellm/router.py index d0f4e5ff44d..15de3d51175 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9948,6 +9948,63 @@ class Router: name for name, fully_blocked in blocked_by_name.items() if fully_blocked } + async def async_get_fully_unhealthy_model_names(self) -> Set[str]: + """ + Returns the set of model names where every backing deployment is currently + marked unhealthy by background health checks (and the health state is not stale). + + Used by `/v1/models?healthy_only=true` to hide models that cannot serve any + request. A model with at least one healthy (or unknown-health) deployment + remains visible. Returns an empty set when no health state is available, so + callers fail open to the unfiltered listing. + + Notes: + - Mirrors `_async_filter_health_check_unhealthy_deployments`: when + `allowed_fails_policy` is set, cooldown is the sole routing exclusion + mechanism, so nothing is hidden here either. + - Team-specific public model names (`team_public_model_name`) are + aggregated alongside `model_name`, so team aliases of fully-unhealthy + deployments are hidden too (unlike `get_fully_blocked_model_names`, + which matches `model_name` only). + - Wildcard routes (e.g. `openai/*`) are matched by their literal + deployment name only; models expanded from a wildcard route are not + hidden (fail open). + - Intentionally diverges from the routing-time safety net (which + bypasses the health filter when every candidate is unhealthy and + still attempts the request): hiding here is presentation-only — + it answers "should this model be advertised?", not "should a + request for it still be attempted?". A hidden model can still be + called directly. + """ + if self.allowed_fails_policy is not None: + return set() + unhealthy_ids = ( + await self.health_state_cache.async_get_unhealthy_deployment_ids() + ) + if not unhealthy_ids: + return set() + deployments = self.get_model_list() or [] + unhealthy_by_name: Dict[str, bool] = {} + for deployment in deployments: + model_info = deployment.get("model_info") or {} + names = [deployment.get("model_name") or ""] + team_public_model_name = model_info.get("team_public_model_name") + if team_public_model_name: + names.append(team_public_model_name) + is_unhealthy = model_info.get("id") in unhealthy_ids + for name in names: + if not name: + continue + if name in unhealthy_by_name: + unhealthy_by_name[name] = unhealthy_by_name[name] and is_unhealthy + else: + unhealthy_by_name[name] = is_unhealthy + return { + name + for name, fully_unhealthy in unhealthy_by_name.items() + if fully_unhealthy + } + def _get_team_specific_model( self, deployment: DeploymentTypedDict, team_id: Optional[str] = None ) -> Optional[str]: diff --git a/tests/test_litellm/proxy/test_model_list_healthy_only.py b/tests/test_litellm/proxy/test_model_list_healthy_only.py new file mode 100644 index 00000000000..4ab33f3bf50 --- /dev/null +++ b/tests/test_litellm/proxy/test_model_list_healthy_only.py @@ -0,0 +1,92 @@ +""" +Tests for the opt-in `healthy_only` filter on GET /v1/models (`model_list`). +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.proxy import proxy_server +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.fixture +def patched_model_list(monkeypatch): + """Stub router + utility helpers used by `model_list`.""" + from litellm.proxy import utils as proxy_utils + + router = MagicMock() + router.get_fully_blocked_model_names = MagicMock(return_value=set()) + router.async_get_fully_unhealthy_model_names = AsyncMock( + return_value={"claude-sonnet"} + ) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "user_model", None) + + async def _fake_get_available_models_for_user(**kwargs): + return ["gpt-4", "claude-sonnet"] + + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + _fake_get_available_models_for_user, + ) + + def _fake_create_model_info_response(model_id, provider="openai", **kwargs): + return {"id": model_id, "object": "model", "created": 0, "owned_by": provider} + + monkeypatch.setattr( + proxy_utils, "create_model_info_response", _fake_create_model_info_response + ) + + return router + + +@pytest.mark.asyncio +async def test_model_list_healthy_only_hides_fully_unhealthy_models( + patched_model_list, +): + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + healthy_only=True, + ) + assert [m["id"] for m in response["data"]] == ["gpt-4"] + + +@pytest.mark.asyncio +async def test_model_list_default_keeps_unhealthy_models(patched_model_list): + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + ) + assert [m["id"] for m in response["data"]] == ["gpt-4", "claude-sonnet"] + patched_model_list.async_get_fully_unhealthy_model_names.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_model_list_healthy_only_applies_to_scope_expand( + patched_model_list, monkeypatch +): + from litellm.proxy.auth import model_checks + from litellm.proxy.management_endpoints import common_utils + + async def _fake_admin(**kwargs): + return True + + monkeypatch.setattr(common_utils, "_user_has_admin_privileges", _fake_admin) + monkeypatch.setattr( + model_checks, + "get_complete_model_list", + lambda **kwargs: ["gpt-4", "claude-sonnet"], + ) + patched_model_list.get_model_names = MagicMock( + return_value=["gpt-4", "claude-sonnet"] + ) + patched_model_list.get_model_access_groups = MagicMock(return_value={}) + + response = await proxy_server.model_list( + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + scope="expand", + healthy_only=True, + ) + assert [m["id"] for m in response["data"]] == ["gpt-4"] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cd235d8de67..49ee871dac6 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -4218,6 +4218,82 @@ def test_get_fully_blocked_model_names_treats_missing_key_as_unblocked(): assert router.get_fully_blocked_model_names() == set() +def _seed_unhealthy_states(router, unhealthy_ids, timestamp=None): + import time + + ts = timestamp if timestamp is not None else time.time() + router.health_state_cache.set_deployment_health_states( + { + uid: {"is_healthy": False, "timestamp": ts, "reason": "test_unhealthy"} + for uid in unhealthy_ids + } + ) + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_marks_name_when_all_unhealthy(): + router = _router_with_two_deployments([False, False]) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}) + assert await router.async_get_fully_unhealthy_model_names() == {"gpt-4o"} + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_keeps_name_when_partial(): + router = _router_with_two_deployments([False, False]) + _seed_unhealthy_states(router, {"dep-0"}) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_empty_without_health_state(): + router = _router_with_two_deployments([False, False]) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_ignores_stale_state(): + import time + + router = _router_with_two_deployments([False, False]) + stale_ts = time.time() - (router.health_state_cache.staleness_threshold + 10) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}, timestamp=stale_ts) + assert await router.async_get_fully_unhealthy_model_names() == set() + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_includes_team_alias(): + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "openai/gpt-4o"}, + "model_info": { + "id": "dep-0", + "team_id": "team-1", + "team_public_model_name": "team-gpt", + }, + } + ] + ) + _seed_unhealthy_states(router, {"dep-0"}) + assert await router.async_get_fully_unhealthy_model_names() == { + "gpt-4o", + "team-gpt", + } + + +@pytest.mark.asyncio +async def test_async_get_fully_unhealthy_model_names_noop_with_allowed_fails_policy(): + from litellm.types.router import AllowedFailsPolicy + + router = _router_with_two_deployments([False, False]) + router.allowed_fails_policy = AllowedFailsPolicy(BadRequestErrorAllowedFails=1) + _seed_unhealthy_states(router, {"dep-0", "dep-1"}) + assert await router.async_get_fully_unhealthy_model_names() == set() + + @pytest.mark.asyncio async def test_async_get_healthy_deployments_skips_blocked_deployment(): router = _router_with_two_deployments([True, False]) From 5b2477bca152b8ad89961af14ab5e99b0b5a5655 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 17 Jun 2026 09:17:22 -0700 Subject: [PATCH 5/9] fix(proxy): list public team model name in /v1/models (#30588) * fix(proxy): optionally surface public team model name in /v1/models Behind general_settings.use_team_public_model_name (default False). When enabled, /v1/models and /models surface the public team_public_model_name for team-scoped (BYOK) models instead of the internal routing key model_name_{team_id}_{uuid} -- consistent with /v1/model/info and OpenAI-compatible. Off by default so the listing's model ids stay backward-compatible for callers that scripted against the internal name; routing by the internal name is unchanged regardless of the flag. Presentation-layer only: access-group, auth, and routing semantics are unchanged; non-team models are pass-through. * fix(proxy): default team model listings to public names * test(proxy): cover team model listing metadata * test(proxy): cover empty team listing deployments * refactor(proxy): simplify team model listing translation * fix(proxy): resolve public team model name on GET /v1/models/{id} The listing endpoints advertise team_public_model_name, but the retrieve endpoint validated and looked up by the raw id, so a public name 404'd. Resolve the public name back to the internal routing key (scoped to the caller's accessible models so colliding names never cross teams), look up by it, and echo the public name back as the response id. * test(proxy): cover public-name resolution on model retrieve * refactor(proxy): extract team model-name translation into TeamModelNameTranslator Move the team-scoped (BYOK) listing/retrieve name translation out of proxy_server.py into a dedicated common_utils module. Static methods with general_settings injected so the logic is unit-testable without globals and proxy_server.py stays thin. * refactor(proxy): use TeamModelNameTranslator in model_list and model_info * test(proxy): target TeamModelNameTranslator for model-name translation * fix(proxy): type create_model_info_response return as dict[str, object] * fix(proxy): keep internal routing key for team model listing metadata lookup Add listing_entries returning (public response id, internal lookup id) so include_metadata=true resolves fallbacks against the routing key the router indexes by, instead of the translated public name (which never matches). * fix(proxy): build /v1/models metadata from internal key, show public id * test(proxy): cover team listing fallback metadata via internal key * fix(proxy): use builtin dict generics in create_model_info_response (UP006) --------- Co-authored-by: Tushar More Co-authored-by: Ishaan Jaffer (cherry picked from commit 60f4c01b741630efb08b451f8cbc6b625835064a) --- .../proxy/common_utils/model_listing_utils.py | 167 +++++ litellm/proxy/proxy_server.py | 67 +- litellm/proxy/utils.py | 58 +- litellm/types/proxy/model_listing.py | 21 + tests/llm_translation/base_llm_unit_tests.py | 5 +- .../test_team_model_name_translation.py | 662 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 18 +- 7 files changed, 946 insertions(+), 52 deletions(-) create mode 100644 litellm/proxy/common_utils/model_listing_utils.py create mode 100644 litellm/types/proxy/model_listing.py diff --git a/litellm/proxy/common_utils/model_listing_utils.py b/litellm/proxy/common_utils/model_listing_utils.py new file mode 100644 index 00000000000..3a70377037d --- /dev/null +++ b/litellm/proxy/common_utils/model_listing_utils.py @@ -0,0 +1,167 @@ +"""Team-scoped (BYOK) model-name translation for the model listing endpoints. + +`/v1/models`, `/models`, and `GET /v1/models/{id}` should surface the public +`team_public_model_name` rather than the internal routing key +`model_name_{team_id}_{uuid}`, consistent with `/v1/model/info`. The internal +key still routes regardless; this is a presentation-layer swap only and does not +touch access-group or auth semantics (see issue #28382). Operators can pin the +legacy internal names with `general_settings.use_team_public_model_name: false`. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from litellm.router import Router + + +class TeamModelNameTranslator: + """Translates internal team routing keys to their public names for the model + listing/retrieve responses. Stateless; the live router and general_settings + are injected per call so the unit tests can drive it without globals. + """ + + @staticmethod + def _internal_public_pair(model: object) -> tuple[str, str] | None: + """`(internal_routing_key, public_name)` for a team-scoped row, else None.""" + if not isinstance(model, dict): + return None + model_dict = cast(dict[str, object], model) # any-ok: checked + model_info_raw: object = model_dict.get("model_info") + if not isinstance(model_info_raw, Mapping): + return None + model_info = cast(Mapping[str, object], model_info_raw) # any-ok: checked + team_id = model_info.get("team_id") + team_public = model_info.get("team_public_model_name") + name = model_dict.get("model_name") + if ( + isinstance(team_id, str) + and isinstance(team_public, str) + and isinstance(name, str) + and team_id + and team_public + and name.startswith(f"model_name_{team_id}_") + ): + return name, team_public + return None + + @staticmethod + def _is_enabled(general_settings: Mapping[str, object]) -> bool: + return general_settings.get("use_team_public_model_name", True) is not False + + @staticmethod + def build_internal_to_public_map( + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> dict[str, str]: + """Internal team routing key -> public `team_public_model_name`. + + Empty when disabled via the legacy flag, the router is absent, or the + router model list is malformed. + """ + if llm_router is None or not TeamModelNameTranslator._is_enabled( + general_settings + ): + return {} + router_model_list = llm_router.get_model_list() + if not isinstance(router_model_list, list): + return {} + return dict( + pair + for pair in ( + TeamModelNameTranslator._internal_public_pair(model) + for model in router_model_list + ) + if pair is not None + ) + + @staticmethod + def _response_to_lookup_map( + model_names: list[str], + internal_to_public: dict[str, str], + ) -> dict[str, str]: + """Map each public response id to the first internal lookup id seen in + `model_names`, preserving first-occurrence order. First-wins keeps list + and retrieve in agreement on which accessible deployment a shared public + id resolves to: a global iterated before a colliding team alias stays + the listed entry, and sibling team rows collapse to their first + occurrence. + """ + result: dict[str, str] = {} + for name in model_names: + result.setdefault(internal_to_public.get(name, name), name) + return result + + @staticmethod + def listing_entries( + model_names: list[str], + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> list[tuple[str, str]]: + """`(response_id, metadata_lookup_id)` for each listed model, de-duplicated + by response_id while preserving order. + + For team-scoped rows `response_id` is the public name shown to the client, + while `metadata_lookup_id` stays the internal routing key so downstream + metadata/fallback lookups (keyed by the routing name) still resolve. The + lookup id is always one of `model_names` (the caller's accessible set), so + a public name shared across teams never resolves to another team's + internal key. Both ids are identical for unmapped names (globals, + access-group keys). + """ + internal_to_public = TeamModelNameTranslator.build_internal_to_public_map( + llm_router, general_settings + ) + if not internal_to_public: + return [(name, name) for name in model_names] + return list( + TeamModelNameTranslator._response_to_lookup_map( + model_names, internal_to_public + ).items() + ) + + @staticmethod + def translate_listing( + model_names: list[str], + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> list[str]: + """Public-name view of `model_names` (the `response_id` of each listing + entry). Sibling deployments sharing a public name collapse to one entry + while preserving order; unmapped names pass through. + """ + return [ + entry[0] + for entry in TeamModelNameTranslator.listing_entries( + model_names, llm_router, general_settings + ) + ] + + @staticmethod + def resolve_public_name( + model_id: str, + available_models: list[str], + llm_router: "Router | None", + general_settings: Mapping[str, object], + ) -> str: + """Resolve a public team name back to the internal routing key the router + indexes by, so `GET /v1/models/{id}` accepts the name the listing returns. + + Resolution is restricted to `available_models` (the caller's accessible + set) so colliding public names across teams never resolve across an access + boundary. Uses the same first-occurrence dedup as `listing_entries` so a + public id advertised by `/v1/models` resolves to the same internal + deployment that the listing's metadata was built from. Returns `model_id` + unchanged when it is not an accessible public team name (already-internal + names and globals pass through). + """ + internal_to_public = TeamModelNameTranslator.build_internal_to_public_map( + llm_router, general_settings + ) + if not internal_to_public: + return model_id + return TeamModelNameTranslator._response_to_lookup_map( + available_models, internal_to_public + ).get(model_id, model_id) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 95e8994ff32..3f7ec4d74fa 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -15,6 +15,7 @@ import threading import time import traceback import warnings +from collections.abc import Mapping from datetime import datetime, timedelta, timezone from typing import ( TYPE_CHECKING, @@ -302,6 +303,7 @@ from litellm.proxy.common_utils.load_config_utils import ( get_config_file_contents_from_gcs, get_file_contents_from_s3, ) +from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) @@ -8228,6 +8230,8 @@ async def model_list( """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj + settings = cast(dict[str, object], general_settings) # any-ok: legacy settings + from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, ) @@ -8307,16 +8311,21 @@ async def model_list( if hidden_names: all_models = [m for m in all_models if m not in hidden_names] - # Build response data with all proxy models + # Surface the public team name by default; legacy internal keys via flag. + # The internal routing key drives the metadata/fallback lookup, while the + # public name is what the client sees as the model id. model_data = [] - for model in all_models: + for response_id, lookup_id in TeamModelNameTranslator.listing_entries( + all_models, llm_router, settings + ): model_info = create_model_info_response( - model_id=model, + model_id=lookup_id, provider="openai", include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, ) + model_info["id"] = response_id model_data.append(model_info) return dict( @@ -8344,16 +8353,21 @@ async def model_list( if hidden_names: all_models = [m for m in all_models if m not in hidden_names] - # Build response data + # Surface the public team name by default; legacy internal keys via flag. + # The internal routing key drives the metadata/fallback lookup, while the + # public name is what the client sees as the model id. model_data = [] - for model in all_models: + for response_id, lookup_id in TeamModelNameTranslator.listing_entries( + all_models, llm_router, settings + ): model_info = create_model_info_response( - model_id=model, + model_id=lookup_id, provider="openai", include_metadata=include_metadata or False, fallback_type=fallback_type, llm_router=llm_router, ) + model_info["id"] = response_id model_data.append(model_info) return dict( @@ -8375,6 +8389,8 @@ async def model_list( async def model_info( model_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + team_id: Optional[str] = None, + healthy_only: Optional[bool] = False, ): """ Retrieve information about a specific model accessible to your API key. @@ -8384,16 +8400,21 @@ async def model_info( Follows OpenAI API specification for individual model retrieval. https://platform.openai.com/docs/api-reference/models/retrieve + + Query parameters mirror `/v1/models` so the same caller context (team + scoping, health filtering, paused deployments) drives both endpoints; the + listing's public id must resolve to the same internal deployment here. """ global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj + settings = cast(dict[str, object], general_settings) # any-ok: legacy settings + from litellm.proxy.utils import ( create_model_info_response, get_available_models_for_user, validate_model_access, ) - # Get available models for the user all_models = await get_available_models_for_user( user_api_key_dict=user_api_key_dict, llm_router=llm_router, @@ -8401,21 +8422,43 @@ async def model_info( user_model=user_model, prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj, - team_id=None, + team_id=team_id, include_model_access_groups=False, only_model_access_groups=False, return_wildcard_routes=False, user_api_key_cache=user_api_key_cache, ) + # Mirror /v1/models' visibility filter so first-occurrence resolution + # cannot land on a deployment the listing had hidden. + blocked_names = ( + llm_router.get_fully_blocked_model_names() if llm_router is not None else set() + ) + unhealthy_names: set[str] = set() + if healthy_only and llm_router is not None: + unhealthy_names = await llm_router.async_get_fully_unhealthy_model_names() + hidden_names = blocked_names | unhealthy_names + if hidden_names: + all_models = [m for m in all_models if m not in hidden_names] + + internal_to_public = TeamModelNameTranslator.build_internal_to_public_map( + llm_router, settings + ) + resolved_model_id = TeamModelNameTranslator.resolve_public_name( + model_id=model_id, + available_models=all_models, + llm_router=llm_router, + general_settings=settings, + ) + # Validate that the requested model is accessible - validate_model_access(model_id=model_id, available_models=all_models) + validate_model_access(model_id=resolved_model_id, available_models=all_models) # Get provider information from the router deployment if llm_router is None: raise HTTPException(status_code=500, detail="Router not initialized") - deployment = llm_router.get_deployment_by_model_group_name(model_id) + deployment = llm_router.get_deployment_by_model_group_name(resolved_model_id) if deployment is None: raise HTTPException( status_code=404, @@ -8425,9 +8468,9 @@ async def model_info( # Use the actual litellm model from the deployment to get provider info _, provider, _, _ = litellm.get_llm_provider(model=deployment.litellm_params.model) - # Return the model information in the same format as the list endpoint + response_id = internal_to_public.get(resolved_model_id, model_id) return create_model_info_response( - model_id=model_id, + model_id=response_id, provider=provider, include_metadata=False, fallback_type=None, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c15c37f6ad8..d86cd38a51c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -45,6 +45,7 @@ from litellm.proxy._types import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.proxy.model_listing import ModelInfoResponse from litellm.types.utils import CallTypes, CallTypesLiteral try: @@ -6231,56 +6232,39 @@ def create_model_info_response( include_metadata: bool = False, fallback_type: Optional[str] = None, llm_router: Optional["Router"] = None, -) -> dict: +) -> ModelInfoResponse: """ - Create a standardized model info response. + Create a standardized OpenAI-compatible model object. - Args: - model_id: The model ID - provider: The model provider - include_metadata: Whether to include metadata - fallback_type: Type of fallbacks to include - llm_router: LiteLLM router instance - - Returns: - Dictionary containing model information + When include_metadata is true, attaches the model's configured fallbacks + (resolved via the router under fallback_type, defaulting to "general"). + Raises HTTPException(400) for an unknown fallback_type. """ from litellm.proxy.auth.model_checks import get_all_fallbacks - model_info = { + base: ModelInfoResponse = { "id": model_id, "object": "model", "created": DEFAULT_MODEL_CREATED_AT_TIME, "owned_by": provider, } + if not include_metadata: + return base - # Add metadata if requested - if include_metadata: - metadata = {} - - # Default fallback_type to "general" if include_metadata is true - effective_fallback_type = ( - fallback_type if fallback_type is not None else "general" + effective_fallback_type = fallback_type if fallback_type is not None else "general" + valid_fallback_types = ("general", "context_window", "content_policy") + if effective_fallback_type not in valid_fallback_types: + raise HTTPException( + status_code=400, + detail=f"Invalid fallback_type. Must be one of: {list(valid_fallback_types)}", ) - # Validate fallback_type - valid_fallback_types = ["general", "context_window", "content_policy"] - if effective_fallback_type not in valid_fallback_types: - raise HTTPException( - status_code=400, - detail=f"Invalid fallback_type. Must be one of: {valid_fallback_types}", - ) - - fallbacks = get_all_fallbacks( - model=model_id, - llm_router=llm_router, - fallback_type=effective_fallback_type, - ) - metadata["fallbacks"] = fallbacks - - model_info["metadata"] = metadata - - return model_info + fallbacks = get_all_fallbacks( + model=model_id, + llm_router=llm_router, + fallback_type=effective_fallback_type, + ) + return {**base, "metadata": {"fallbacks": fallbacks}} def validate_model_access( diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py new file mode 100644 index 00000000000..c3330da0d66 --- /dev/null +++ b/litellm/types/proxy/model_listing.py @@ -0,0 +1,21 @@ +"""Response types for the model listing/retrieve endpoints (/v1/models, /models).""" + +from typing import Literal + +from typing_extensions import NotRequired, TypedDict + + +class ModelInfoMetadata(TypedDict): + fallbacks: list[str] + + +class ModelInfoResponse(TypedDict): + """OpenAI-compatible model object. `metadata` is present only when the + endpoint is called with include_metadata=true. + """ + + id: str + object: Literal["model"] + created: int + owned_by: str + metadata: NotRequired[ModelInfoMetadata] diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index fef1d23d867..a184798b503 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -906,7 +906,10 @@ class BaseLLMChatTest(ABC): { "type": "image_url", "image_url": { - "url": "https://www.gstatic.com/webp/gallery/1.webp", + # sha-pinned in-repo logo via jsdelivr; gstatic's + # robots.txt blocks server-side fetchers (e.g. + # Anthropic), which 400s the request. + "url": "https://cdn.jsdelivr.net/gh/BerriAI/litellm@d769e81c90d453240c61fc572cdb27fae06a89d0/ui/litellm-dashboard/public/assets/logos/litellm_logo.jpg", "detail": detail, }, }, diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py index 6a8e0d15d8b..0f87fcda588 100644 --- a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -15,6 +15,7 @@ import pytest import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils.model_listing_utils import TeamModelNameTranslator from litellm.proxy.proxy_server import ( _get_proxy_model_info, _translate_model_name_for_response, @@ -593,3 +594,664 @@ async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkey team_filter.assert_awaited_once() assert team_filter.await_args.kwargs["team_id"] == "other-team" assert team_filter.await_args.kwargs["all_models"] == [team_row] + + +@pytest.mark.asyncio +async def test_v1_models_translates_team_model_for_access_group_key(monkeypatch): + """Regression (#28382 sibling leak): a virtual key whose model access group + resolves to a team BYOK deployment must list the PUBLIC name in /v1/models, + not the internal routing key model_name_{team_id}_{uuid}. + + The /model/info read-path fix did not cover /v1/models, which builds from + bare model-name strings via access-group expansion. + """ + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + # Default behavior: listing surfaces public names. + monkeypatch.setattr(ps, "general_settings", {}) + + # virtual key granted access via the access group (no team membership) + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key) + + ids = [d["id"] for d in resp["data"]] + assert "tushar-gpt-4.1" in ids + assert "model_name_teamX_uuid9" not in ids + + +@pytest.mark.asyncio +async def test_v1_models_keeps_internal_names_when_public_name_flag_disabled( + monkeypatch, +): + """Compatibility override: /v1/models can still list the internal routing + name for consumers that scripted against those ids. Translation is enabled + by default and disabled via general_settings['use_team_public_model_name']. + """ + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {"use_team_public_model_name": False}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key) + + ids = [d["id"] for d in resp["data"]] + assert "model_name_teamX_uuid9" in ids # internal id preserved (backward-compat) + assert "tushar-gpt-4.1" not in ids + + +@pytest.mark.asyncio +async def test_v1_models_translates_team_model_with_metadata(monkeypatch): + """include_metadata=true must build metadata for the public model id.""" + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key, include_metadata=True) + + assert resp["data"] == [ + { + "id": "tushar-gpt-4.1", + "object": "model", + "created": 1677610602, + "owned_by": "openai", + "metadata": {"fallbacks": []}, + } + ] + + +@pytest.mark.asyncio +async def test_v1_models_metadata_fallbacks_use_internal_routing_key(monkeypatch): + """Regression: with include_metadata=true, fallbacks configured for a team + model under its internal routing key must still surface. The metadata lookup + has to run against the internal name, not the translated public name (which + the router's fallback config never keys on) -- otherwise fallbacks silently + drop to [].""" + team_dep = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "id1", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_dep] + router.get_model_list.return_value = [team_dep] + # Fallbacks are keyed on the internal routing name, as the router stores them. + router.fallbacks = [{"model_name_teamX_uuid9": ["gpt-4o-backup"]}] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key, include_metadata=True) + + assert resp["data"] == [ + { + "id": "tushar-gpt-4.1", + "object": "model", + "created": 1677610602, + "owned_by": "openai", + "metadata": {"fallbacks": ["gpt-4o-backup"]}, + } + ] + + +@pytest.mark.asyncio +async def test_v1_models_metadata_does_not_leak_other_team_fallbacks(monkeypatch): + """Regression: two teams can publish the same team_public_model_name. With + include_metadata=true a caller scoped to teamX must see teamX's fallbacks for + the shared public name, never teamY's. The metadata lookup has to stay within + the caller's accessible models; resolving the public name through a router-wide + reverse map could point it at another team's internal routing key.""" + team_x = { + "model_name": "model_name_teamX_uuid9", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "idX", + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + "access_groups": ["grp-a"], + }, + } + team_y = { + "model_name": "model_name_teamY_uuidZ", + "litellm_params": {"model": "azure/gpt-4.1"}, + "model_info": { + "id": "idY", + "team_id": "teamY", + "team_public_model_name": "tushar-gpt-4.1", # same public name, other team + }, + } + router = MagicMock() + router.get_model_names.return_value = ["model_name_teamX_uuid9"] + router.get_model_access_groups.return_value = {"grp-a": ["model_name_teamX_uuid9"]} + router.get_fully_blocked_model_names.return_value = set() + router.model_list = [team_x, team_y] + router.get_model_list.return_value = [team_x, team_y] + router.fallbacks = [ + {"model_name_teamX_uuid9": ["teamX-backup"]}, + {"model_name_teamY_uuidZ": ["teamY-backup"]}, + ] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "general_settings", {}) + + key = UserAPIKeyAuth( + user_id="u", api_key="sk-test", models=["grp-a"], team_models=[] + ) + resp = await ps.model_list(user_api_key_dict=key, include_metadata=True) + + assert resp["data"] == [ + { + "id": "tushar-gpt-4.1", + "object": "model", + "created": 1677610602, + "owned_by": "openai", + "metadata": {"fallbacks": ["teamX-backup"]}, + } + ] + + +def test_translate_team_model_names_for_listing_swaps_and_dedupes(): + """Internal team routing keys -> public name; sibling deployments sharing a + public name collapse to one entry (order preserved); globals untouched.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + { + "model_name": "model_name_teamX_uuidB", # sibling: same public name + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + {"model_name": "gpt-4o", "model_info": {"db_model": False}}, + ] + + out = TeamModelNameTranslator.translate_listing( + ["model_name_teamX_uuidA", "model_name_teamX_uuidB", "gpt-4o"], + router, + {}, + ) + assert out == ["tushar-gpt-4.1", "gpt-4o"] + + +def test_listing_entries_keep_internal_lookup_id_for_team_rows(): + """`listing_entries` returns (public response id, internal lookup id) so the + response shows the public name while metadata lookups keep the routing key. + Sibling deployments collapse to one entry; globals map to themselves.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + { + "model_name": "model_name_teamX_uuidB", # sibling: same public name + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + {"model_name": "gpt-4o", "model_info": {"db_model": False}}, + ] + + entries = TeamModelNameTranslator.listing_entries( + ["model_name_teamX_uuidA", "model_name_teamX_uuidB", "gpt-4o"], + router, + {}, + ) + # public id for the client; an internal routing key for the metadata lookup + assert entries[0][0] == "tushar-gpt-4.1" + assert entries[0][1].startswith("model_name_teamX_uuid") + assert entries[1] == ("gpt-4o", "gpt-4o") + assert len(entries) == 2 + + +def test_listing_entries_lookup_id_never_crosses_team_boundary(): + """Regression: when two teams share a team_public_model_name, the lookup id for + the shared public name must stay within the caller's accessible model_names and + never resolve to the other team's internal routing key (which would leak that + team's fallback metadata under include_metadata=true).""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "shared-name", + }, + }, + { + "model_name": "model_name_teamY_uuidB", # different team, same public name + "model_info": { + "team_id": "teamY", + "team_public_model_name": "shared-name", + }, + }, + ] + + # caller can only access teamX's internal key + entries = TeamModelNameTranslator.listing_entries( + ["model_name_teamX_uuidA"], router, {} + ) + + assert entries == [("shared-name", "model_name_teamX_uuidA")] + + +def test_listing_entries_global_wins_when_team_alias_collides_with_global(): + """Regression: when an accessible global model shares its name with a team + deployment's `team_public_model_name`, the listing must keep the global + entry rather than overwriting its lookup id with the colliding team's + internal routing key (which would surface the team's metadata under the + global id).""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "gpt-4o", + }, + }, + {"model_name": "gpt-4o", "model_info": {"db_model": False}}, + ] + + entries = TeamModelNameTranslator.listing_entries( + ["gpt-4o", "model_name_teamX_uuidA"], router, {} + ) + + assert entries == [("gpt-4o", "gpt-4o")] + + +def test_listing_and_resolve_agree_on_sibling_internal_key(): + """Regression: when two team deployments share a public name, listing and + retrieve must pick the same internal routing key, otherwise `/v1/models/{id}` + describes a different deployment than what the listing's metadata was built + from.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + { + "model_name": "model_name_teamX_uuidB", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + }, + ] + available = ["model_name_teamX_uuidA", "model_name_teamX_uuidB"] + + [(_, listing_lookup)] = TeamModelNameTranslator.listing_entries( + available, router, {} + ) + resolve_lookup = TeamModelNameTranslator.resolve_public_name( + model_id="tushar-gpt-4.1", + available_models=available, + llm_router=router, + general_settings={}, + ) + + assert listing_lookup == resolve_lookup + + +def test_listing_entries_skips_empty_team_public_model_name(): + """Regression: a misconfigured row with `team_public_model_name: ""` must not + produce a listing entry with an empty `id`; the internal routing key should + pass through unchanged, matching `/v1/model/info`'s falsy-check behavior.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "", + }, + }, + ] + + entries = TeamModelNameTranslator.listing_entries( + ["model_name_teamX_uuidA"], router, {} + ) + + assert entries == [("model_name_teamX_uuidA", "model_name_teamX_uuidA")] + + +def test_listing_entries_passthrough_when_disabled(): + """Legacy flag / no router -> response id equals lookup id (no translation).""" + assert TeamModelNameTranslator.listing_entries(["a", "b"], None, {}) == [ + ("a", "a"), + ("b", "b"), + ] + + +def test_translate_team_model_names_for_listing_leaves_unmapped_names(): + """Names with no team mapping (globals, access-group keys) pass through.""" + router = MagicMock() + router.get_model_list.return_value = [ + {"model_name": "gpt-4o", "model_info": {"db_model": False}} + ] + + assert TeamModelNameTranslator.translate_listing( + ["gpt-4o", "beta-group"], router, {} + ) == ["gpt-4o", "beta-group"] + + +def test_translate_team_model_names_for_listing_none_router(): + """No router -> return the input list unchanged.""" + assert TeamModelNameTranslator.translate_listing(["a", "b"], None, {}) == ["a", "b"] + + +def test_translate_team_model_names_for_listing_respects_legacy_flag(): + """Operators can keep returning the legacy internal routing key.""" + router = MagicMock() + router.get_model_list.return_value = [ + { + "model_name": "model_name_teamX_uuidA", + "model_info": { + "team_id": "teamX", + "team_public_model_name": "tushar-gpt-4.1", + }, + } + ] + + assert TeamModelNameTranslator.translate_listing( + ["model_name_teamX_uuidA"], router, {"use_team_public_model_name": False} + ) == ["model_name_teamX_uuidA"] + + +def _public_named_router(*team_rows: dict) -> MagicMock: + router = MagicMock() + router.get_model_list.return_value = list(team_rows) + return router + + +def test_resolve_public_name_to_internal_routing_key(): + """A public team name resolves back to the internal routing key the router + indexes by, so `GET /v1/models/{public_name}` can find the deployment.""" + router = _public_named_router(_team_row()) + + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={}, + ) + == "model_name_team-abc-123_4a6b8" + ) + + +def test_resolve_public_name_is_access_scoped_across_teams(): + """Two teams can publish the SAME public name. A caller's query must resolve + to the internal key they can actually access, never another team's.""" + # both rows share public name "team-claude-sonnet" + router = _public_named_router(_team_row(), _other_team_row()) + + # caller only has access to their own team's internal key + resolved = TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={}, + ) + assert resolved == "model_name_team-abc-123_4a6b8" + assert resolved != "model_name_team-other_9f2c1" + + +def test_resolve_public_name_unmapped_passes_through(): + """A public name with no accessible internal mapping is returned unchanged so + the caller hits the normal 404/access path; internal names pass through too.""" + router = _public_named_router(_team_row()) + + # not accessible -> unchanged (downstream validate_model_access will 404) + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=[], + llm_router=router, + general_settings={}, + ) + == "team-claude-sonnet" + ) + # already an internal routing key -> unchanged + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="model_name_team-abc-123_4a6b8", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={}, + ) + == "model_name_team-abc-123_4a6b8" + ) + + +def test_resolve_public_name_respects_legacy_flag(): + """With the legacy flag set, no public-name resolution happens.""" + router = _public_named_router(_team_row()) + + assert ( + TeamModelNameTranslator.resolve_public_name( + model_id="team-claude-sonnet", + available_models=["model_name_team-abc-123_4a6b8"], + llm_router=router, + general_settings={"use_team_public_model_name": False}, + ) + == "team-claude-sonnet" + ) + + +@pytest.mark.asyncio +async def test_retrieve_model_by_public_name_returns_200(monkeypatch): + """Regression: `GET /v1/models/{public_name}` must NOT 404. The listing + advertises the public team name, so retrieve must accept the same name, + resolve it to the internal routing key for lookup, and echo the public name + back as the model id.""" + import litellm + import litellm.proxy.utils as proxy_utils + + team_row = _team_row() + router = _public_named_router(team_row) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]), + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + resp = await ps.model_info(model_id="team-claude-sonnet", user_api_key_dict=key) + + assert resp["id"] == "team-claude-sonnet" + # lookup happened by the internal routing key, not the public name + router.get_deployment_by_model_group_name.assert_called_once_with( + "model_name_team-abc-123_4a6b8" + ) + + +@pytest.mark.asyncio +async def test_retrieve_model_by_internal_name_returns_public_id(monkeypatch): + """Regression: retrieving by the internal routing key must echo the SAME + public id `/v1/models` advertises for that deployment, not the path. Otherwise + a client iterating the listing's id and then retrieving each one would observe + a different id depending on which alias they queried by.""" + import litellm + import litellm.proxy.utils as proxy_utils + + router = _public_named_router(_team_row()) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]), + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + resp = await ps.model_info( + model_id="model_name_team-abc-123_4a6b8", user_api_key_dict=key + ) + + assert resp["id"] == "team-claude-sonnet" + + +@pytest.mark.asyncio +async def test_retrieve_model_by_internal_name_keeps_internal_id_when_flag_disabled( + monkeypatch, +): + """With `use_team_public_model_name=false`, retrieve must keep the internal + routing key as the response id, mirroring `/v1/models`' legacy output.""" + import litellm + import litellm.proxy.utils as proxy_utils + + router = _public_named_router(_team_row()) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {"use_team_public_model_name": False}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=["model_name_team-abc-123_4a6b8"]), + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + resp = await ps.model_info( + model_id="model_name_team-abc-123_4a6b8", user_api_key_dict=key + ) + + assert resp["id"] == "model_name_team-abc-123_4a6b8" + + +@pytest.mark.asyncio +async def test_retrieve_model_by_inaccessible_public_name_404s(monkeypatch): + """A caller without access to a team model still gets 404 when retrieving by + its public name; resolution never crosses the access boundary.""" + import litellm + import litellm.proxy.utils as proxy_utils + + router = _public_named_router(_team_row()) + deployment = MagicMock() + deployment.litellm_params.model = "azure/gpt-5.2-low-rpm-testing" + router.get_deployment_by_model_group_name.return_value = deployment + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "general_settings", {}) + monkeypatch.setattr( + proxy_utils, + "get_available_models_for_user", + AsyncMock(return_value=[]), # caller has no access + ) + monkeypatch.setattr( + litellm, "get_llm_provider", lambda model: (model, "openai", None, None) + ) + + key = UserAPIKeyAuth(user_id="u", api_key="sk-test", team_models=[]) + with pytest.raises(ps.HTTPException) as exc_info: + await ps.model_info(model_id="team-claude-sonnet", user_api_key_dict=key) + + assert exc_info.value.status_code == 404 + router.get_deployment_by_model_group_name.assert_not_called() diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index c5560ca7f20..9699da9a6ef 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -6279,6 +6279,10 @@ export interface paths { * * Follows OpenAI API specification for individual model retrieval. * https://platform.openai.com/docs/api-reference/models/retrieve + * + * Query parameters mirror `/v1/models` so the same caller context (team + * scoping, health filtering, paused deployments) drives both endpoints; the + * listing's public id must resolve to the same internal deployment here. */ get: operations["model_info_models__model_id__get"]; put?: never; @@ -14472,6 +14476,10 @@ export interface paths { * * Follows OpenAI API specification for individual model retrieval. * https://platform.openai.com/docs/api-reference/models/retrieve + * + * Query parameters mirror `/v1/models` so the same caller context (team + * scoping, health filtering, paused deployments) drives both endpoints; the + * listing's public id must resolve to the same internal deployment here. */ get: operations["model_info_v1_models__model_id__get"]; put?: never; @@ -38145,7 +38153,10 @@ export interface operations { }; model_info_models__model_id__get: { parameters: { - query?: never; + query?: { + team_id?: string | null; + healthy_only?: boolean | null; + }; header?: never; path: { model_id: string; @@ -48211,7 +48222,10 @@ export interface operations { }; model_info_v1_models__model_id__get: { parameters: { - query?: never; + query?: { + team_id?: string | null; + healthy_only?: boolean | null; + }; header?: never; path: { model_id: string; From 9e3098529046fd16b5050b0557ff8496bf0229c8 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Wed, 17 Jun 2026 10:32:52 -0700 Subject: [PATCH 6/9] fix(proxy): resolve list files credentials from team BYOK deployments (#30495) * fix(proxy): resolve list files credentials from team BYOK deployments GET /v1/files without target_model_names now prefers the team's own deployment (model_info.team_id) over shared global provider keys, so JWT team auth lists files against the correct upstream account. Co-authored-by: Cursor * fix(proxy): scope list files credential lookup to team allowlist Remove the unrestricted deployment scan that could leak global provider keys to teams without access, normalize all-proxy-models to the team-scoped model list, and fix TID251 violations by using dict instead of Dict/Any. Co-authored-by: Cursor --------- Co-authored-by: Cursor (cherry picked from commit 6c8b60d50d9985e5e56e09e1081a63d445598e17) --- .../openai_files_endpoints/common_utils.py | 88 +++++ .../openai_files_endpoints/files_endpoints.py | 25 +- .../test_files_endpoint.py | 326 ++++++++++++++++++ 3 files changed, 436 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index cc0d06e4f40..694924f2b90 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -10,6 +10,8 @@ from litellm.types.utils import SpecialEnums if TYPE_CHECKING: from fastapi import Request + from litellm.router import Router + def _is_base64_encoded_unified_file_id(b64_uid: str) -> Union[str, Literal[False]]: # Ensure b64_uid is a string and not a mock object @@ -296,6 +298,92 @@ def get_credentials_for_model( return credentials +def get_team_provider_credentials( + llm_router: Optional["Router"], + team_models: List[str], + custom_llm_provider: str, + team_id: Optional[str] = None, +) -> Optional[dict]: + """ + Resolve upstream credentials for a provider-scoped file operation + (e.g. GET /v1/files), which doesn't pin a model. + + Priority: + 1. The team's own (BYOK) deployment for this provider — a deployment whose + ``model_info.team_id`` matches ``team_id``. This keeps team-scoped listings + on the team's own provider account/key instead of a shared global one. + 2. Fallback: any deployment the team is granted access to for this provider, + expanding wildcard routes and the all-proxy-models sentinel. + + Credential lookup is always scoped to the team's allowlist, so a team can + never resolve a provider key for a deployment it isn't authorized to use. + Returns None when the router is unavailable or no authorized deployment + matches, so the caller can fall back to default credential resolution. + """ + if llm_router is None: + return None + + def _provider_credentials(model_id: str) -> Optional[dict]: + credentials = llm_router.get_deployment_credentials_with_provider( + model_id=model_id + ) + if ( + credentials is not None + and credentials.get("custom_llm_provider") == custom_llm_provider + ): + return credentials + return None + + # 1. Prefer the team's own BYOK deployment, matched by model_info.team_id. + if team_id is not None: + for deployment in llm_router.model_list or []: + model_info = deployment.get("model_info") or {} + if model_info.get("team_id") != team_id: + continue + deployment_id = model_info.get("id") + if deployment_id is None: + continue + credentials = _provider_credentials(deployment_id) + if credentials is not None: + return credentials + + # 2. Fall back to deployments the team is allowed to access. The + # all-proxy-models sentinel isn't expanded by get_complete_model_list, so + # normalize it to an empty allowlist, which defers to the team-scoped + # proxy model list. A team with a restricted allowlist (e.g. anthropic + # only) therefore never resolves another provider's key. + from litellm.proxy._types import SpecialModelNames + from litellm.proxy.auth.model_checks import get_complete_model_list + + grants_all_models = SpecialModelNames.all_proxy_models.value in team_models + effective_team_models = [] if grants_all_models else team_models + + proxy_model_list = llm_router.get_model_names(team_id=team_id) + model_access_groups = llm_router.get_model_access_groups() + models_to_try = list( + dict.fromkeys( + get_complete_model_list( + key_models=[], + team_models=effective_team_models, + proxy_model_list=proxy_model_list, + user_model=None, + infer_model_from_keys=False, + return_wildcard_routes=True, + llm_router=llm_router, + model_access_groups=model_access_groups, + include_model_access_groups=True, + team_id=team_id, + ) + ) + ) + for model_name in models_to_try: + credentials = _provider_credentials(model_name) + if credentials is not None: + return credentials + + return None + + def prepare_data_with_credentials( data: dict, credentials: dict, diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 378cbbda89c..57b6d111e0e 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -51,6 +51,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( encode_file_id_with_model, extract_file_creation_params, get_credentials_for_model, + get_team_provider_credentials, handle_model_based_routing, prepare_data_with_credentials, ) @@ -1344,14 +1345,20 @@ async def list_files( status_code=400, detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", ) - ## Use router to list fine-tuning jobs for that model if llm_router is None: raise HTTPException( status_code=500, detail="LLM Router not initialized. Ensure models added to proxy.", ) - data["model"] = target_model_names_list[0] - response = await llm_router.afile_list( + credentials = get_credentials_for_model( + llm_router=llm_router, + model_id=target_model_names_list[0], + operation_context="file list", + ) + prepare_data_with_credentials(data=data, credentials=credentials) + response = await litellm.afile_list( + custom_llm_provider=credentials["custom_llm_provider"], + purpose=purpose, **data, ) else: @@ -1363,6 +1370,18 @@ async def list_files( or "openai" ) + # No model/target_model_names pinned: resolve upstream credentials from + # the team's deployment for this provider so the call is authenticated + # against the team's own account (e.g. the team's openai deployment). + team_credentials = get_team_provider_credentials( + llm_router=llm_router, + team_models=user_api_key_dict.team_models or [], + custom_llm_provider=custom_llm_provider, + team_id=user_api_key_dict.team_id, + ) + if team_credentials is not None: + prepare_data_with_credentials(data=data, credentials=team_credentials) + response = await litellm.afile_list( custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore ) diff --git a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py index 5fc36b71f2b..007c04e0aff 100644 --- a/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/test_litellm/proxy/openai_files_endpoint/test_files_endpoint.py @@ -1873,3 +1873,329 @@ def test_get_file_content_non_openai_provider_skips_streaming_handler( assert "stream" not in captured_kwargs mock_streaming_response.assert_not_awaited() proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_resolves_wildcard_deployment_credentials( + mocker: MockerFixture, monkeypatch +): + """ + GET /v1/files?target_model_names= must resolve the upstream api_key + from the matching (wildcard) deployment. Regression for the path routing + through llm_router.afile_list(model=...), which reached OpenAI without an + api_key and failed with "api_key client option must be set". + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + wildcard_router = Router( + model_list=[ + { + "model_name": "*", + "litellm_params": { + "model": "openai/*", + "api_key": "wildcard-openai-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + + try: + response = client.get( + "/v1/files?target_model_names=gpt-4o", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "wildcard-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_without_target_model_names_uses_team_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Plain GET /v1/files (no target_model_names) must resolve the upstream openai + api_key from the team's openai deployment instead of falling through to a + keyless OpenAI client. Regression for "api_key client option must be set". + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + wildcard_router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=["openai/*"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_restricted_team_does_not_leak_global_openai_credentials( + mocker: MockerFixture, monkeypatch +): + """ + A team whose allowlist only grants anthropic must NOT resolve a global + openai deployment's api_key for plain GET /v1/files. Regression for the + last-resort scan that ignored team access control. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "global-openai-key", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="anthropic-only-team", + team_models=["claude-opus-4-6"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") != "global-openai-key" + + +def test_list_files_prefers_team_byok_over_global_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + When a team has its own BYOK openai deployment (model_info.team_id set), plain + GET /v1/files must use the team's key, not a shared/global openai deployment. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "global-openai-key", + }, + }, + { + "model_name": "team-gpt-4o", + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "team-byok-openai-key", + }, + "model_info": { + "id": "team-byok-deployment-id", + "team_id": "test-team", + "team_public_model_name": "team-gpt-4o", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=["team-gpt-4o"], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-byok-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() + + +def test_list_files_with_all_proxy_models_team_uses_openai_deployment( + mocker: MockerFixture, monkeypatch +): + """ + Teams with all-proxy-models (or empty models) must still resolve openai + credentials for plain GET /v1/files. + """ + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles, SpecialModelNames + + wildcard_router = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": { + "model": "openai/*", + "api_key": "team-openai-key", + }, + }, + { + "model_name": "claude-opus-4-6", + "litellm_params": { + "model": "anthropic/claude-opus-4-6", + "api_key": "anthropic-key", + }, + }, + ] + ) + + proxy_logging_obj = setup_proxy_logging_object(monkeypatch, wildcard_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", wildcard_router) + proxy_logging_obj.update_request_status = mocker.AsyncMock() + proxy_logging_obj.post_call_success_hook = mocker.AsyncMock(return_value=[]) + proxy_logging_obj.post_call_failure_hook = mocker.AsyncMock() + + captured_kwargs: dict = {} + + async def _mock_afile_list(**kwargs): + captured_kwargs.update(kwargs) + return [] + + monkeypatch.setattr(litellm, "afile_list", _mock_afile_list) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="test-user", + team_id="test-team", + team_models=[SpecialModelNames.all_proxy_models.value], + ) + + try: + response = client.get( + "/v1/files", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert captured_kwargs.get("api_key") == "team-openai-key" + assert captured_kwargs.get("custom_llm_provider") == "openai" + proxy_logging_obj.post_call_failure_hook.assert_not_called() From 0dc6951af6a1e1817e236f47827efe709f930a2b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:36:23 -0700 Subject: [PATCH 7/9] fix(cost): stop non-string service_tier from silently dropping cost tracking (#30690) completion_cost read service_tier straight from the request optional_params and called service_tier.lower() on it, so a non-string value (dict/int/list, reachable via allowed_openai_params/drop_params) raised AttributeError. _response_cost_calculator swallowed that and returned response_cost=None, so the request's cost was silently lost. The isinstance guard alone is not enough: a surviving dict would crash again downstream in _get_service_tier_cost_key, which also calls .lower(). A request-level service_tier is only meaningful for pricing when it is a concrete billable tier string, so coerce any non-string value to None and defer to the tier the provider reports on the response usage, the same way "auto" already does. Adds a regression test driving a dict service_tier through completion_cost; it raises AttributeError before the fix and prices at the served tier after. (cherry picked from commit 43dadc5138d4d267176e11fe5a47d0f0593d790d) --- litellm/cost_calculator.py | 12 +++-- tests/test_litellm/test_cost_calculator.py | 52 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6c11c8d8a06..71a2577092b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1227,10 +1227,14 @@ def completion_cost( # noqa: PLR0915 if service_tier is None and optional_params is not None: service_tier = optional_params.get("service_tier") - # "auto" is a routing preference, not a billable tier: the provider picks - # the tier and reports the one actually served on the response/usage, so - # defer to that instead of pricing the request-level "auto" as standard - if service_tier is not None and service_tier.lower() == ServiceTier.AUTO.value: + # A request-level service_tier only prices the request when it is a + # concrete billable tier string. "auto" is a routing preference and any + # non-string value is not a billable tier, so defer to the tier the + # provider reports on the response/usage instead of crashing or mispricing + if ( + not isinstance(service_tier, str) + or service_tier.lower() == ServiceTier.AUTO.value + ): service_tier = None # Extract service_tier from completion_response if not provided diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index bcc685f0366..ff4e36f593c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -2054,6 +2054,58 @@ def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(): assert cost == pytest.approx(expected_priority) +def test_completion_cost_non_string_service_tier_defers_to_served_tier(): + """ + Regression: a non-string request-level ``service_tier`` (reachable via + ``allowed_openai_params``/``drop_params``) must not crash cost tracking. + + Before the fix, ``completion_cost`` called ``service_tier.lower()`` on the + request-level value, so a dict raised ``AttributeError``. ``_response_cost_calculator`` + swallowed it and reported ``response_cost=None``, silently dropping the cost. + The non-string preference must be ignored so pricing defers to the tier the + provider actually served on the response usage. + """ + from litellm import completion_cost + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "claude-test-non-string-tier-cost-model" + litellm.register_model( + model_cost={ + model: { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "input_cost_per_token_priority": 6e-6, + "output_cost_per_token_priority": 30e-6, + "litellm_provider": "anthropic", + "max_tokens": 8192, + } + } + ) + + usage = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 1000, + "output_tokens": 500, + "service_tier": "priority", + }, + reasoning_content=None, + ) + response = ModelResponse(usage=usage, model=model) + + cost = completion_cost( + completion_response=response, + model=model, + custom_llm_provider="anthropic", + optional_params={"service_tier": {"name": "auto"}}, + ) + + expected_priority = 1000 * 6e-6 + 500 * 30e-6 + assert cost == pytest.approx(expected_priority) + + def test_anthropic_cost_per_token_prices_cache_at_served_tier_with_multiplier(): """ Regression for the cache/tier interaction in the Anthropic geo/speed path. From 01d3593cbd0c0e1ad5bd6d3be3607f9819e436b5 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 17 Jun 2026 19:02:32 -0700 Subject: [PATCH 8/9] =?UTF-8?q?bump:=20version=201.89.1=20=E2=86=92=201.89?= =?UTF-8?q?.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4a456b2c9a7..b5ef93f033c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.89.1" +version = "1.89.2" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.14" @@ -264,7 +264,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.89.1" +version = "1.89.2" version_files = [ "pyproject.toml:^version", ] From ad758c978843f5b2950ca5f44ac3ed576728cdec Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Wed, 17 Jun 2026 19:02:33 -0700 Subject: [PATCH 9/9] chore: refresh uv.lock for 1.89.2 --- uv.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uv.lock b/uv.lock index 81e099a7f06..efda9444fed 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-06-13T01:42:46.429412Z" +exclude-newer = "2026-06-15T02:02:32.823508Z" exclude-newer-span = "P3D" [manifest] @@ -3280,7 +3280,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.89.1" +version = "1.89.2" source = { editable = "." } dependencies = [ { name = "aiohttp" },