From 7f1b0a46334c2a8bdda029f1d7e83db5f4c55d36 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:04:44 +0000 Subject: [PATCH 01/34] feat(bedrock): add us-gov.xai.grok-4.7 GovCloud pricing (#50288) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 25 +++++++++++++++++++ model_prices_and_context_window.json | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1bda9d24c8e..53685e3cce0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76502,6 +76502,31 @@ "supports_tool_choice": true, "supports_vision": true }, + "us-gov.xai.grok-4.7": { + "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-7.html", + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": { "cache_read_input_token_cost": 1.4e-08, "input_cost_per_token": 6e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1bda9d24c8e..53685e3cce0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76502,6 +76502,31 @@ "supports_tool_choice": true, "supports_vision": true }, + "us-gov.xai.grok-4.7": { + "supports_regex_lookaround": false, + "supported_endpoints": ["/v1/chat/completions"], + "supports_bedrock_runtime_chat_completions_tools_with_reasoning": true, + "supports_bedrock_runtime_chat_completions_response_format": true, + "cache_read_input_token_cost": 6.6e-07, + "cache_read_input_token_cost_flex": 3.3e-07, + "cache_read_input_token_cost_priority": 1.155e-06, + "input_cost_per_token": 2.64e-06, + "input_cost_per_token_flex": 1.32e-06, + "input_cost_per_token_priority": 4.62e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 524288, + "max_output_tokens": 524288, + "mode": "chat", + "output_cost_per_token": 7.92e-06, + "output_cost_per_token_flex": 3.96e-06, + "output_cost_per_token_priority": 1.386e-05, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-7.html", + "supports_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "baseten/deepseek-ai/DeepSeek-V4.1-Flash-Fast": { "cache_read_input_token_cost": 1.4e-08, "input_cost_per_token": 6e-07, From 4a092f06f0775ce00cea6af35413d054b3285045 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:26:45 +0000 Subject: [PATCH 02/34] fix(bedrock): add GovCloud Grok 4.7 batch prices (#50291) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 4 +++- model_prices_and_context_window.json | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 53685e3cce0..3bb6eb725b3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76511,6 +76511,7 @@ "cache_read_input_token_cost_flex": 3.3e-07, "cache_read_input_token_cost_priority": 1.155e-06, "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, "input_cost_per_token_flex": 1.32e-06, "input_cost_per_token_priority": 4.62e-06, "litellm_provider": "bedrock_converse", @@ -76518,9 +76519,10 @@ "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, "output_cost_per_token_flex": 3.96e-06, "output_cost_per_token_priority": 1.386e-05, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-7.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 53685e3cce0..3bb6eb725b3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76511,6 +76511,7 @@ "cache_read_input_token_cost_flex": 3.3e-07, "cache_read_input_token_cost_priority": 1.155e-06, "input_cost_per_token": 2.64e-06, + "input_cost_per_token_batches": 1.32e-06, "input_cost_per_token_flex": 1.32e-06, "input_cost_per_token_priority": 4.62e-06, "litellm_provider": "bedrock_converse", @@ -76518,9 +76519,10 @@ "max_output_tokens": 524288, "mode": "chat", "output_cost_per_token": 7.92e-06, + "output_cost_per_token_batches": 3.96e-06, "output_cost_per_token_flex": 3.96e-06, "output_cost_per_token_priority": 1.386e-05, - "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-xai-grok-4-7.html", + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-gov-east-1/index.json", "supports_function_calling": true, "supports_prompt_caching": false, "supports_reasoning": true, From 300db9f01dd6bb840e062b92bb22f76b516c0079 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Thu, 5 Nov 2026 15:34:08 -0800 Subject: [PATCH 03/34] test(router): expect no fallback errors header for non-bool include_fallback_errors (#50259) The Rust core reports fallback errors only when include_fallback_errors is literally true. The routing and SDK wire tests still expected the header for truthy non-bool values (1 and a 5 KB string), which the deleted Python core produced by accident. Every non-bool value now expects no errors, and the flag still must not reach the provider. --- .../test_include_fallback_errors_wire.py | 22 +++++++++---------- ...est_router_include_fallback_errors_wire.py | 18 +++++++-------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/tests/integration/routing/test_include_fallback_errors_wire.py b/tests/integration/routing/test_include_fallback_errors_wire.py index 9075f5e41d3..2f48bc43a22 100644 --- a/tests/integration/routing/test_include_fallback_errors_wire.py +++ b/tests/integration/routing/test_include_fallback_errors_wire.py @@ -61,10 +61,10 @@ _PROVIDERS: Final = MappingProxyType( ) _STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream")) _NON_BOOLEAN_FLAGS: Final = ( - pytest.param(1, True, id="int"), - pytest.param("", False, id="empty-string"), - pytest.param([], False, id="list"), - pytest.param("x" * 5120, True, id="five-kilobytes"), + pytest.param(1, id="int"), + pytest.param("", id="empty-string"), + pytest.param([], id="list"), + pytest.param("x" * 5120, id="five-kilobytes"), ) _OPENAI_UNAUTHORIZED: Final = Reply( status=401, @@ -326,10 +326,8 @@ def test_default_gateway_keeps_the_flag_off_the_wire_on_a_fallback( assert observed.attempted == "1", observed -@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) -def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire( - registered: _Registered, value: JsonValue, errors_reported: bool -) -> None: +@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS) +def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire(registered: _Registered, value: JsonValue) -> None: sent: Final = _send( registered.gateway, _PATHS["chat"], @@ -497,9 +495,9 @@ def test_exposed_chat_without_a_fallback_keeps_the_flag_off_the_wire_and_the_err assert observed.errors_header is None, observed -@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) -def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_header_follows_its_truthiness( - exposed: _Exposed, value: JsonValue, errors_reported: bool +@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS) +def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_reports_no_errors( + exposed: _Exposed, value: JsonValue ) -> None: sent: Final = _send( exposed.proxy, @@ -515,7 +513,7 @@ def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_heade assert observed.leaks == (), observed assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed assert observed.attempted == "1", observed - assert _errors_mention_the_outage(observed.errors_header) is errors_reported, observed + assert observed.errors_header is None, observed def test_exposed_proxy_rejects_an_unauthenticated_flagged_request_before_the_wire(exposed: _Exposed) -> None: diff --git a/tests/integration/sdk/test_router_include_fallback_errors_wire.py b/tests/integration/sdk/test_router_include_fallback_errors_wire.py index fb17e05b02a..a8952c8c2b5 100644 --- a/tests/integration/sdk/test_router_include_fallback_errors_wire.py +++ b/tests/integration/sdk/test_router_include_fallback_errors_wire.py @@ -36,11 +36,11 @@ _BURST: Final = 12 _CALLBACK_WINDOW: Final = 15.0 _STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream")) _NON_BOOLEAN_FLAGS: Final = ( - pytest.param(1, True, id="int"), - pytest.param("", False, id="empty-string"), - pytest.param([], False, id="list"), - pytest.param("x" * 5120, True, id="five-kilobytes"), - pytest.param(False, False, id="false"), + pytest.param(1, id="int"), + pytest.param("", id="empty-string"), + pytest.param([], id="list"), + pytest.param("x" * 5120, id="five-kilobytes"), + pytest.param(False, id="false"), ) _UNAUTHORIZED: Final = Reply( status=401, @@ -263,10 +263,8 @@ def test_without_a_fallback_the_flag_still_stays_off_the_wire() -> None: assert outcome.errors == (), outcome -@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) -def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its_truthiness( - value: object, errors_reported: bool -) -> None: +@pytest.mark.parametrize("value", _NON_BOOLEAN_FLAGS) +def test_a_flag_other_than_true_stays_off_the_wire_and_reports_no_errors(value: object) -> None: with wire_server(_peer) as wire: router: Final = _router(wire) outcome: Final = _outcome(wire, _complete(router, "primary", stream=False, include_fallback_errors=value)) @@ -274,7 +272,7 @@ def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its assert outcome.hit == ("primary", "backup"), outcome assert outcome.text == f"{_ANSWER} backup", outcome assert outcome.attempted == 1, outcome - assert (len(outcome.errors) == 1) is errors_reported, outcome + assert outcome.errors == (), outcome class _Recorder(CustomLogger): From e011a4ea9b70978075fe52f36867c89d1c18ad2f Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Thu, 5 Nov 2026 15:34:31 -0800 Subject: [PATCH 04/34] fix(router): do not cool down a deployment on a 404 from a response id lookup (#50262) The Rust core records every attempt failure against the routed deployment, so a client GET, DELETE, cancel or input_items call on a response id that the provider answers with 404 counted toward cooldown. The removed Python core never saw these failures. Exempt them in failure_exits next to the background poll 404 and video lookup exits. --- litellm/router_rust/surface/health.py | 2 + litellm/router_utils/cooldown_handlers.py | 20 ++++++++++ .../router_utils/test_cooldown_handlers.py | 23 +++++++++++ .../test_router_provider_endpoints.py | 38 +++++++++++++++++++ 4 files changed, 83 insertions(+) diff --git a/litellm/router_rust/surface/health.py b/litellm/router_rust/surface/health.py index b16adc38af9..0653518d0d7 100644 --- a/litellm/router_rust/surface/health.py +++ b/litellm/router_rust/surface/health.py @@ -279,6 +279,7 @@ class RustHealthCallbacks: is_caller_scoped_auth_failure, is_caller_timeout_408, is_failure_outside_selected_deployment, + is_response_lookup_not_found, is_video_lookup_not_found, ) @@ -288,6 +289,7 @@ class RustHealthCallbacks: is_failure_outside_selected_deployment(exception if isinstance(exception, BaseException) else None) or (isinstance(exception, Exception) and is_background_response_poll_not_found(exception, litellm_params)) or is_video_lookup_not_found(exception, logged) + or is_response_lookup_not_found(exception, logged) or is_caller_timeout_408(logged, status) # pyright: ignore[reportArgumentType] # the status as raised or is_caller_scoped_auth_failure(self._deployment_of(deployment_id), status) # pyright: ignore[reportArgumentType] # see above ) diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 03485ebd5f8..4c918776223 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -108,6 +108,26 @@ def is_video_lookup_not_found(exception: object, failure_kwargs: Mapping[str, ob return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in VIDEO_ID_CALL_TYPES +# The calls on an existing response id. ``compact_responses`` is left out: it sends a model and input, +# so its 404 can mean a missing model rather than a missing response. +RESPONSE_ID_CALL_TYPES: Final = frozenset( + { + "aget_responses", + "get_responses", + "adelete_responses", + "delete_responses", + "acancel_responses", + "cancel_responses", + "alist_input_items", + } +) + + +def is_response_lookup_not_found(exception: object, failure_kwargs: Mapping[str, object]) -> bool: + """Whether a call on an existing response id failed with a provider 404: the response is not on that account.""" + return getattr(exception, "status_code", None) == 404 and failure_kwargs.get("call_type") in RESPONSE_ID_CALL_TYPES + + _EXCEPTION_POLICY_FIELDS: Final[tuple[tuple[type, str], ...]] = ( # ContentPolicyViolationError subclasses BadRequestError, so it must be checked first. (litellm.ContentPolicyViolationError, "ContentPolicyViolationErrorAllowedFails"), diff --git a/tests/unit/router_utils/test_cooldown_handlers.py b/tests/unit/router_utils/test_cooldown_handlers.py index 8ef683c0aee..6a35c8a9602 100644 --- a/tests/unit/router_utils/test_cooldown_handlers.py +++ b/tests/unit/router_utils/test_cooldown_handlers.py @@ -36,6 +36,7 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments, cast_exception_status_to_int, get_cooldown_deployments, + is_response_lookup_not_found, mark_advisor_orchestration_failure, set_cooldown_deployments, should_cooldown_based_on_allowed_fails_policy, @@ -3565,3 +3566,25 @@ def test_the_refusal_names_each_cooled_deployment_some_cooldown_state_routes_the with pytest.raises(RouterRateLimitError) as refusal: _select(_tenant_router(EVERY_TENANT, **BOTH_ON), "async", "tenants", copy.deepcopy(request_kwargs)) assert {deployment["model_id"] for deployment in refusal.value.cooldown_deployments or ()} == reachable + + +@pytest.mark.parametrize( + ("status_code", "call_type", "expected"), + [ + (404, "aget_responses", True), + (404, "adelete_responses", True), + (404, "acancel_responses", True), + (404, "alist_input_items", True), + (404, "get_responses", True), + (404, "aresponses", False), + (404, "acompact_responses", False), + (404, "acompletion", False), + (500, "aget_responses", False), + (429, "alist_input_items", False), + ], +) +def test_is_response_lookup_not_found(status_code: int, call_type: str, expected: bool) -> None: + error: Final = litellm.NotFoundError(message="gone", llm_provider="openai", model="gpt-4o-mini") + error.status_code = status_code + + assert is_response_lookup_not_found(error, {"call_type": call_type}) is expected diff --git a/tests/unit/test_router/test_router_provider_endpoints.py b/tests/unit/test_router/test_router_provider_endpoints.py index a78228b8658..baf3e94f1f0 100644 --- a/tests/unit/test_router/test_router_provider_endpoints.py +++ b/tests/unit/test_router/test_router_provider_endpoints.py @@ -18,6 +18,7 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.proxy._types import UserAPIKeyAuth +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.router_callbacks.track_deployment_metrics import ( get_deployment_successes_for_current_minute, ) @@ -921,6 +922,43 @@ async def test_video_lookup_404_never_cools_down_the_deployment_that_answered(re assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == [] +@pytest.mark.parametrize( + ("endpoint", "method", "path"), + [ + ("aget_responses", "GET", ""), + ("adelete_responses", "DELETE", ""), + ("acancel_responses", "POST", "/cancel"), + ("alist_input_items", "GET", "/input_items"), + ], +) +@pytest.mark.asyncio +async def test_response_lookup_404_never_cools_down_the_deployment_that_answered( + respx_mock: respx.MockRouter, endpoint: str, method: str, path: str +) -> None: + respx_mock.route(method=method, url=f"https://api.openai.com/v1/responses/resp_gone{path}").respond( + 404, json={"error": {"code": "NotFound", "message": "Response not found"}} + ) + router: Final = Router( + model_list=[ + { + "model_name": "gpt", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": api_key}, + "model_info": {"id": deployment_id}, + } + for api_key, deployment_id in (("key-a", "dep-a"), ("key-b", "dep-b")) + ], + num_retries=0, + allowed_fails=1, + ) + response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id("openai", "dep-b", "resp_gone") + + for _ in range(3): + with pytest.raises(litellm.NotFoundError): + await getattr(router, endpoint)(response_id=response_id) + + assert await router.cooldown_cache.async_get_active_cooldowns(["dep-a", "dep-b"], parent_otel_span=None) == [] + + @pytest.mark.asyncio async def test_chat_404_still_cools_down_the_deployment(respx_mock: respx.MockRouter) -> None: respx_mock.post("https://api.openai.com/v1/chat/completions").respond( From 4679b9f642ce8521b645c57a83249aaa81676758 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:38:33 -0800 Subject: [PATCH 05/34] test(copilot): align login guards with direct access tokens (#50285) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/device_login.py | 64 +++++++++++++--- .../test_device_code_login_guard_boot.py | 14 ++-- .../test_device_code_login_guard_wire.py | 74 ++++++++++--------- .../sdk/test_device_code_login_guard_sdk.py | 4 +- 4 files changed, 99 insertions(+), 57 deletions(-) diff --git a/tests/integration/_support/device_login.py b/tests/integration/_support/device_login.py index 55519678f40..55eb5a753bb 100644 --- a/tests/integration/_support/device_login.py +++ b/tests/integration/_support/device_login.py @@ -25,6 +25,7 @@ from pydantic import JsonValue CHATGPT_AUTH_HOST: Final = "auth.openai.com" GITHUB_HOST: Final = "github.com" GITHUB_API_HOST: Final = "api.github.com" +COPILOT_API_HOST: Final = "api.githubcopilot.com" AUTH_HOSTS: Final = (CHATGPT_AUTH_HOST, GITHUB_HOST, GITHUB_API_HOST) CHATGPT_ACCOUNT: Final = "acct-device-login" _FAR_FUTURE: Final = 4102444800 @@ -79,7 +80,17 @@ COPILOT_MINTED_KEY: Final = "tid=device-login;minted" COPILOT_REJECTED_KEY: Final = "tid=device-login;rejected" CONTROL_KEY: Final = "device-login-control-key" ACCEPTED_BEARERS: Final = frozenset( - {CHATGPT_STORED, CHATGPT_REFRESHED, CHATGPT_FIRST_LOGIN, COPILOT_STORED_KEY, COPILOT_MINTED_KEY, CONTROL_KEY} + { + CHATGPT_STORED, + CHATGPT_REFRESHED, + CHATGPT_FIRST_LOGIN, + COPILOT_ACCESS, + COPILOT_FIRST_LOGIN_ACCESS, + COPILOT_UNSAVED_KEY_ACCESS, + COPILOT_STORED_KEY, + COPILOT_MINTED_KEY, + CONTROL_KEY, + } ) REJECTED_DETAIL: Final = "the scripted provider rejected this bearer token" _DEVICE_AUTH_ID: Final = "device-auth-synthetic" @@ -180,6 +191,10 @@ def _auth_hosts(api_url: str, grant: threading.Event) -> Callable[[Request], Rep return _json(200, {"access_token": COPILOT_FIRST_LOGIN_ACCESS, "token_type": "bearer"}) case ("api.github.com", "GET", "/copilot_internal/v2/token"): return _copilot_api_key(request, api_url) + case ("api.github.com", "GET", "/copilot_internal/user"): + if request.headers.get("authorization") == f"token {COPILOT_REJECTED_ACCESS}": + return _json(401, {"message": "Bad credentials"}) + return _json(200, {"endpoints": {"api": f"https://{COPILOT_API_HOST}"}}) case _: return _switched_off() @@ -204,9 +219,22 @@ class Switches: switch.clear() +@dataclass(frozen=True, slots=True) +class _ProviderAPI: + plain: Wire + secure: Wire + + @property + def url(self) -> str: + return self.plain.url + + def drain(self) -> tuple[Request, ...]: + return (*self.plain.drain(), *self.secure.drain()) + + @dataclass(frozen=True, slots=True) class Peers: - api: Wire + api: _ProviderAPI auth: Wire tunnel: str cert: Path @@ -228,6 +256,14 @@ class Peers: def auth_connections(self) -> tuple[str, ...]: return tuple(self.authorities.get_nowait() for _ in range(self.authorities.qsize())) + def assert_only_copilot_discovery(self, access_token: str) -> None: + assert set(self.auth_connections()) <= {f"{GITHUB_API_HOST}:443"} + requests: Final = self.auth.drain() + assert { + (request.method, request.target, request.headers.get("host"), request.headers.get("authorization")) + for request in requests + } <= {("GET", "/copilot_internal/user", GITHUB_API_HOST, f"token {access_token}")}, requests + def dropped(self) -> tuple[Hangup, ...]: return tuple(self.hangups.get_nowait() for _ in range(self.hangups.qsize())) @@ -279,9 +315,8 @@ class _TunnelServer(socketserver.ThreadingTCPServer): @contextmanager def _tunnel( - destination: Wire, switches: Switches, authorities: SimpleQueue[str], hangups: SimpleQueue[Hangup] + destination: Wire, provider: Wire, switches: Switches, authorities: SimpleQueue[str], hangups: SimpleQueue[Hangup] ) -> Generator[str]: - destination_port: Final = int(destination.url.rsplit(":", 1)[1]) class Tunnel(socketserver.StreamRequestHandler): rbufsize = 0 @@ -306,20 +341,24 @@ def _tunnel( if len(request_line) < 2: return authority: Final = request_line[1] - if authority.rsplit(":", 1)[0] not in AUTH_HOSTS: + host: Final = authority.rsplit(":", 1)[0] + if host not in (*AUTH_HOSTS, COPILOT_API_HOST): self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n") return - authorities.put(authority) - if switches.refuse_connect.is_set(): + is_auth: Final = host in AUTH_HOSTS + if is_auth: + authorities.put(authority) + if is_auth and switches.refuse_connect.is_set(): self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\ncontent-length: 0\r\n\r\n") return - if self.hold(authority, opened, switches.hold_connect) is None: + if is_auth and self.hold(authority, opened, switches.hold_connect) is None: return self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") - client_hello: Final = self.hold(authority, opened, switches.stall_tls) + client_hello: Final = self.hold(authority, opened, switches.stall_tls) if is_auth else b"" if client_hello is None: return self.request.settimeout(10) + destination_port: Final = int((destination if is_auth else provider).url.rsplit(":", 1)[1]) with socket.create_connection(("127.0.0.1", destination_port), timeout=10) as upstream: upstream.sendall(client_hello) outbound: Final = threading.Thread(target=_pipe, args=(self.request, upstream)) @@ -340,16 +379,17 @@ def _tunnel( @contextmanager def device_login_peers(directory: Path) -> Generator[Peers]: - cert, key = write_self_signed_cert(directory, AUTH_HOSTS) + cert, key = write_self_signed_cert(directory, (*AUTH_HOSTS, COPILOT_API_HOST)) switches: Final = Switches(threading.Event(), threading.Event(), threading.Event(), threading.Event()) authorities: Final[SimpleQueue[str]] = SimpleQueue() hangups: Final[SimpleQueue[Hangup]] = SimpleQueue() with ( wire_server(_provider_api(rv.ResponsesVendor())) as api, + wire_server(_provider_api(rv.ResponsesVendor()), tls=server_context(cert, key)) as copilot_api, wire_server(_auth_hosts(api.url, switches.grant), tls=server_context(cert, key)) as auth, - _tunnel(auth, switches, authorities, hangups) as tunnel, + _tunnel(auth, copilot_api, switches, authorities, hangups) as tunnel, ): - yield Peers(api, auth, tunnel, cert, authorities, hangups, switches) + yield Peers(_ProviderAPI(api, copilot_api), auth, tunnel, cert, authorities, hangups, switches) def chatgpt_record( diff --git a/tests/integration/providers/test_device_code_login_guard_boot.py b/tests/integration/providers/test_device_code_login_guard_boot.py index 91f502179f3..384d6635bb0 100644 --- a/tests/integration/providers/test_device_code_login_guard_boot.py +++ b/tests/integration/providers/test_device_code_login_guard_boot.py @@ -67,9 +67,9 @@ class _TokenDirs: def environment(self) -> Mapping[str, str]: return MappingProxyType({"CHATGPT_TOKEN_DIR": str(self.chatgpt), "GITHUB_COPILOT_TOKEN_DIR": str(self.copilot)}) - def store_valid_tokens(self, api_url: str) -> None: + def store_valid_tokens(self) -> None: dl.write_chatgpt(self.chatgpt, dl.chatgpt_record(dl.CHATGPT_STORED)) - dl.write_copilot_key(self.copilot, dl.COPILOT_STORED_KEY, api_url) + dl.write_copilot_access(self.copilot, dl.COPILOT_ACCESS) def _token_dirs(directory: Path) -> _TokenDirs: @@ -242,16 +242,16 @@ def test_booting_without_tokens_lists_login_deployments_and_serves_only_after_to assert peers.auth_connections() == () assert dl.bearer(peers.api.drain()[-1]) == dl.CONTROL_KEY - dirs.store_valid_tokens(peers.api.url) + dirs.store_valid_tokens() _reload(owned) _every_worker_lists(owned, workers) for served_logins, name in enumerate(_LOGIN_NAMES, start=1): served: Final = _chat(_base_url(owned), key, name, uuid.uuid4().hex) assert served.status == 200, served.text _successes_logged(key, at_least=_CONTROL_CALLS_BEFORE_THE_RELOAD + served_logins) - assert peers.auth_connections() == () + peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) forwarded: Final = peers.api.drain() - assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_STORED, dl.COPILOT_STORED_KEY} + assert {dl.bearer(request) for request in forwarded} == {dl.CHATGPT_STORED, dl.COPILOT_ACCESS} dl.write_chatgpt(dirs.chatgpt, dl.chatgpt_record(dl.CHATGPT_EXPIRED, refresh_token=dl.GOOD_REFRESH)) peers.switches.hold_connect.set() @@ -287,7 +287,7 @@ def test_a_token_removed_after_boot_turns_every_request_into_the_refusal( ) -> None: dirs: Final = _token_dirs(tmp_path) with dl.device_login_peers(tmp_path) as peers: - dirs.store_valid_tokens(peers.api.url) + dirs.store_valid_tokens() with owned_proxy_process( gateway, tmp_path, @@ -310,7 +310,7 @@ def test_a_token_removed_after_boot_turns_every_request_into_the_refusal( refused: Final = _chat(_base_url(owned), key, name, uuid.uuid4().hex) assert refused.status == 400, refused.text assert refusal in refused.message(), refused.text - assert peers.auth_connections() == () + peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) tail: Final = owned.log.read_text() assert dl.CHATGPT_USER_CODE not in tail and dl.COPILOT_USER_CODE not in tail, tail[-3000:] control: Final = _chat(_base_url(owned), key, _CONTROL, uuid.uuid4().hex) diff --git a/tests/integration/providers/test_device_code_login_guard_wire.py b/tests/integration/providers/test_device_code_login_guard_wire.py index 39e36fd95fa..9074ea83189 100644 --- a/tests/integration/providers/test_device_code_login_guard_wire.py +++ b/tests/integration/providers/test_device_code_login_guard_wire.py @@ -2,7 +2,6 @@ import asyncio import hashlib import itertools import json -import re import threading import time import uuid @@ -26,9 +25,10 @@ from integration._support.database import read_rows from integration._support.process import OwnedProxy, owned_proxy_process from integration._support.timing import assert_cut_short, timed from integration._support.wire import Request -from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS from pydantic import JsonValue +from litellm.llms.chatgpt.authenticator import DEVICE_CODE_COOLDOWN_SECONDS + pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state] Provider: TypeAlias = Literal["chatgpt", "copilot"] @@ -52,7 +52,7 @@ _REFUSALS: Final[Mapping[Provider, str]] = MappingProxyType( {"chatgpt": dl.CHATGPT_REFUSAL, "copilot": dl.COPILOT_REFUSAL} ) _STORED_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType( - {"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_STORED_KEY} + {"chatgpt": dl.CHATGPT_STORED, "copilot": dl.COPILOT_ACCESS} ) _CONTROL_MODEL: Final = "device-login-control" _EXTRA: Final[Mapping[str, JsonValue]] = MappingProxyType({"num_retries": 0, "cache": {"no-cache": True}}) @@ -154,7 +154,7 @@ class _Scene: if provider == "chatgpt": dl.write_chatgpt(self.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_STORED)) return - dl.write_copilot_key(self.rig.copilot_dir, dl.COPILOT_STORED_KEY, self.rig.peers.api.url) + dl.write_copilot_access(self.rig.copilot_dir, dl.COPILOT_ACCESS) def _config(peers: dl.Peers, directory: Path) -> Path: @@ -403,7 +403,7 @@ def test_missing_token_is_refused_with_the_login_instructions_before_any_auth_ho @pytest.mark.parametrize("cell", _CELLS, ids=_Cell.name) -def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene, cell: _Cell) -> None: +def test_stored_token_serves_the_request_without_a_device_login(scene: _Scene, cell: _Cell) -> None: scene.store_valid_token(cell.provider) call: Final = scene.call(cell) served: Final = _serve(call) @@ -412,7 +412,10 @@ def test_stored_token_serves_the_request_without_an_auth_host_call(scene: _Scene received: Final = scene.rig.peers.api.drain() (forwarded,) = _carrying(received, call.marker) assert dl.bearer(forwarded) == _STORED_BEARERS[cell.provider], _describe(received) - assert scene.rig.peers.auth_connections() == () + if cell.provider == "copilot": + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) + else: + assert scene.rig.peers.auth_connections() == () (row,) = _spend_rows(scene.key) assert row["status"] == "success", row @@ -487,7 +490,7 @@ def test_missing_token_is_refused_on_the_other_model_routes(scene: _Scene, route assert _describe(scene.rig.peers.api.drain()) == () -def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) -> None: +def test_copilot_embeddings_are_served_with_the_stored_access_token(scene: _Scene) -> None: scene.store_valid_token("copilot") served: Final = scene.post( "/v1/embeddings", {"model": "github_copilot/text-embedding-3-small", "input": "login", **_EXTRA} @@ -496,9 +499,9 @@ def test_copilot_embeddings_are_served_with_the_stored_api_key(scene: _Scene) -> (item,) = rv.ITEMS.validate_python(rv.JSON_OBJECT.validate_json(served.text)["data"]) assert item["embedding"] == [0.25, 0.5, 0.75], served.text (forwarded,) = scene.rig.peers.api.drain() - assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_STORED_KEY), forwarded.target + assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_ACCESS), forwarded.target assert forwarded.target.endswith("/embeddings"), forwarded.target - assert scene.rig.peers.auth_connections() == () + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) @pytest.mark.parametrize("provider", _PROVIDERS) @@ -562,71 +565,67 @@ def test_stored_token_the_provider_rejects_surfaces_its_401_without_a_login( if provider == "chatgpt": dl.write_chatgpt(scene.rig.chatgpt_dir, dl.chatgpt_record(dl.CHATGPT_REJECTED)) else: - dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url) + dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS) call: Final = scene.call(_Cell(provider, endpoint, False, "httpx")) served: Final = _serve(call) assert served.status == 401, served.text assert dl.REJECTED_DETAIL in served.text, served.text received: Final = scene.rig.peers.api.drain() assert {dl.bearer(request) for request in _carrying(received, call.marker)} == { - dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_KEY + dl.CHATGPT_REJECTED if provider == "chatgpt" else dl.COPILOT_REJECTED_ACCESS }, _describe(received) - assert scene.rig.peers.auth_connections() == () + if provider == "copilot": + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_REJECTED_ACCESS) + else: + assert scene.rig.peers.auth_connections() == () @pytest.mark.parametrize("stale_key", (False, True), ids=("missing-api-key", "expired-api-key")) -def test_copilot_api_key_is_minted_once_from_the_stored_access_token(scene: _Scene, stale_key: bool) -> None: +def test_copilot_access_token_is_forwarded_without_minting_an_api_key(scene: _Scene, stale_key: bool) -> None: dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_ACCESS) if stale_key: dl.write_copilot_key(scene.rig.copilot_dir, dl.COPILOT_REJECTED_KEY, scene.rig.peers.api.url, -3600) + legacy_key: Final = scene.rig.copilot_dir / "api-key.json" + stored_key: Final = legacy_key.read_bytes() if stale_key else None call: Final = scene.call(_Cell("copilot", "chat", False, "httpx")) served: Final = _serve(call) assert served.status == 200, served.text assert set(rv.MARKER.findall(served.text)) == {call.marker}, served.text - (minted,) = scene.rig.peers.auth.drain() - assert (minted.method, minted.target, minted.headers.get("host"), minted.headers.get("authorization")) == ( - "GET", - "/copilot_internal/v2/token", - dl.GITHUB_API_HOST, - f"token {dl.COPILOT_ACCESS}", - ) + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker) - assert dl.bearer(forwarded) == dl.COPILOT_MINTED_KEY - assert json.loads((scene.rig.copilot_dir / "api-key.json").read_text())["token"] == dl.COPILOT_MINTED_KEY + assert dl.bearer(forwarded) == dl.COPILOT_ACCESS + assert (legacy_key.read_bytes() if legacy_key.exists() else None) == stored_key -def test_copilot_api_key_that_cannot_be_saved_is_minted_at_most_once_per_worker(scene: _Scene) -> None: +def test_copilot_access_token_serves_when_the_legacy_api_key_path_is_a_directory(scene: _Scene) -> None: dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_UNSAVED_KEY_ACCESS) unwritable_key_file: Final = scene.rig.copilot_dir / "api-key.json" unwritable_key_file.mkdir() first: Final = scene.call(_Cell("copilot", "chat", False, "httpx")) first_served: Final = _serve(first) assert first_served.status == 200, first_served.text - (minted,) = scene.rig.peers.auth.drain() - assert minted.headers.get("authorization") == f"token {dl.COPILOT_UNSAVED_KEY_ACCESS}" later: Final = tuple(scene.call(_Cell("copilot", "chat", False, "httpx")) for _ in range(3)) later_served: Final = tuple(_serve(call) for call in later) assert [served.status for served in later_served] == [200, 200, 200], [served.text for served in later_served] - assert len(scene.rig.peers.auth.drain()) <= 1 + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_UNSAVED_KEY_ACCESS) received: Final = scene.rig.peers.api.drain() forwarded: Final = tuple(_carrying(received, call.marker) for call in (first, *later)) assert [tuple(dl.bearer(request) for request in requests) for requests in forwarded] == [ - (dl.COPILOT_MINTED_KEY,) + (dl.COPILOT_UNSAVED_KEY_ACCESS,) ] * 4, _describe(received) assert unwritable_key_file.is_dir() def test_copilot_access_token_github_rejects_fails_the_request_without_a_device_login(scene: _Scene) -> None: dl.write_copilot_access(scene.rig.copilot_dir, dl.COPILOT_REJECTED_ACCESS) - served: Final = _serve(scene.call(_Cell("copilot", "chat", False, "httpx"))) - assert served.status == 400, served.text - assert "Failed to refresh API key" in _error_message(served.text), served.text + call: Final = scene.call(_Cell("copilot", "chat", False, "httpx")) + served: Final = _serve(call) + assert served.status == 401, served.text + assert dl.REJECTED_DETAIL in _error_message(served.text), served.text _failure_logged(served) - reached: Final = scene.rig.peers.auth.drain() - assert {(request.method, request.target) for request in reached} == {("GET", "/copilot_internal/v2/token")}, ( - _describe(reached) - ) - assert _describe(scene.rig.peers.api.drain()) == () + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_REJECTED_ACCESS) + (forwarded,) = _carrying(scene.rig.peers.api.drain(), call.marker) + assert (forwarded.method, dl.bearer(forwarded)) == ("POST", dl.COPILOT_REJECTED_ACCESS) def test_expired_copilot_api_key_with_no_access_token_is_refused_before_any_auth_host_call(scene: _Scene) -> None: @@ -727,7 +726,10 @@ def test_a_token_mounted_after_a_refusal_serves_without_a_restart_and_removing_i after: Final = _serve(scene.call(cell)) assert after.status == 400, after.text assert _REFUSALS[provider] in _error_message(after.text), after.text - assert scene.rig.peers.auth_connections() == () + if provider == "copilot": + scene.rig.peers.assert_only_copilot_discovery(dl.COPILOT_ACCESS) + else: + assert scene.rig.peers.auth_connections() == () _COOLDOWN_SECONDS_LEFT: Final = 45 diff --git a/tests/integration/sdk/test_device_code_login_guard_sdk.py b/tests/integration/sdk/test_device_code_login_guard_sdk.py index 7640fe13d5d..256871faf7c 100644 --- a/tests/integration/sdk/test_device_code_login_guard_sdk.py +++ b/tests/integration/sdk/test_device_code_login_guard_sdk.py @@ -184,7 +184,7 @@ def clean(bench: _Bench) -> Iterator[_Bench]: _FIRST_LOGIN_BEARERS: Final[Mapping[Provider, str]] = MappingProxyType( - {"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_MINTED_KEY} + {"chatgpt": dl.CHATGPT_FIRST_LOGIN, "copilot": dl.COPILOT_FIRST_LOGIN_ACCESS} ) @@ -205,7 +205,7 @@ def test_sync_call_on_the_main_thread_keeps_the_interactive_first_login( assert {dl.bearer(request) for request in forwarded} == {_FIRST_LOGIN_BEARERS[provider]}, [ (request.method, request.target) for request in forwarded ] - expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token", "api-key.json") + expected_files: Final = ("auth.json",) if provider == "chatgpt" else ("access-token",) assert clean.token_files() == expected_files From 3b20b484732955a7231da98825747ab75385fc19 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 5 Nov 2026 15:38:44 -0800 Subject: [PATCH 06/34] feat(interactions): forward continuation_token and type requires_continuation so callers can resume a long decode (#50151) * feat(interactions): continue requires_continuation long decodes with continuation_token hops A foreground Interactions request stops at Google's per-hop output cap with status requires_continuation and a continuation_token. litellm now re-posts the request with previous_interaction_id and continuation_token until the status moves on, merging the hops into one interaction (last id, every hop's steps, summed usage with initial_input_tokens from hop 1), for non-streaming and streaming calls. continuation_token is accepted as a request param, and requires_continuation is a known status for billing. * refactor(gemini interactions): validate streaming chunks with model_validate * fix(interactions): keep the held completion across stream hops, stop on a repeated continuation_token, follow the spec redirect * fix(interactions): add grounding_tool_count per tool type, stop on any seen continuation_token, keep streaming past a [DONE] frame when a hop is pending * ci(interactions): point the spec canary at aistudio.google.com, where Google moved the Interactions OpenAPI spec * test(interactions): scripted Google ends each long decode hop with a [DONE] frame * refactor(interactions): pass requires_continuation and continuation_token through instead of looping hops in the proxy The caller drives the continuation, the same as against Google directly. /interactions accepts continuation_token in place of input so a resume request needs no input. * test(proxy): assert the accepted /interactions bodies pass the required-param check * fix(proxy): keep the router default fallback for input on /interactions, continuation_token stands in for it --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../workflows/interactions-spec-canary.yml | 4 +- .../interactions/background_cost_polling.py | 4 +- litellm/interactions/main.py | 5 + litellm/interactions/utils.py | 1 + .../gemini/interactions/transformation.py | 4 +- litellm/proxy/route_llm_request.py | 9 +- litellm/types/interactions/generated.py | 24 + .../test_gemini_interactions_wire.py | 79 ++- .../interactions/interactions.openapi.json | 551 ++++++++++-------- tests/unit/interactions/test_main.py | 86 ++- .../interactions/test_openapi_compliance.py | 8 +- .../hooks/test_proxy_track_cost_callback.py | 1 + tests/unit/proxy/test_route_llm_request.py | 31 + 13 files changed, 547 insertions(+), 260 deletions(-) diff --git a/.github/workflows/interactions-spec-canary.yml b/.github/workflows/interactions-spec-canary.yml index a2e4646dce5..89ebdd87a7c 100644 --- a/.github/workflows/interactions-spec-canary.yml +++ b/.github/workflows/interactions-spec-canary.yml @@ -49,9 +49,9 @@ jobs: id: compliance run: | uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \ - --interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \ + --interactions-spec-url=https://aistudio.google.com/api-reference/interactions.openapi.json \ --tb=short -vv -p no:rerunfailures -p no:pytest-retry \ - --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev + --allow-hosts=127.0.0.1,::1,localhost,ai.google.dev,aistudio.google.com tracking-issue: name: tracking-issue diff --git a/litellm/interactions/background_cost_polling.py b/litellm/interactions/background_cost_polling.py index 0b8c128286a..d3cc04408b6 100644 --- a/litellm/interactions/background_cost_polling.py +++ b/litellm/interactions/background_cost_polling.py @@ -71,12 +71,12 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj _TERMINAL_STATUSES: Final = frozenset( - {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"} + {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action", "requires_continuation"} ) _POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"}) -_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"}) +_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action", "requires_continuation"}) SettlementOutcome: TypeAlias = Literal["billed", "released", "unsettled"] diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index acd43c9b830..cd144646f79 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -95,6 +95,7 @@ async def acreate( response_mime_type: str | None = None, # Continuation previous_interaction_id: str | None = None, + continuation_token: str | None = None, # Extra params extra_headers: dict[str, str] | None = None, extra_body: Mapping[str, object] | None = None, @@ -129,6 +130,7 @@ async def acreate( response_format: JSON schema for response format response_mime_type: MIME type of the response previous_interaction_id: ID of previous interaction for continuation + continuation_token: Token from a `requires_continuation` interaction to resume its long decode extra_headers: Additional headers extra_body: Additional body parameters timeout: Request timeout @@ -164,6 +166,7 @@ async def acreate( response_format=response_format, response_mime_type=response_mime_type, previous_interaction_id=previous_interaction_id, + continuation_token=continuation_token, extra_headers=extra_headers, extra_body=extra_body, timeout=timeout, @@ -224,6 +227,7 @@ def create( response_mime_type: str | None = None, # Continuation previous_interaction_id: str | None = None, + continuation_token: str | None = None, # Extra params extra_headers: dict[str, str] | None = None, extra_body: Mapping[str, object] | None = None, @@ -262,6 +266,7 @@ def create( response_format: JSON schema for response format response_mime_type: MIME type of the response previous_interaction_id: ID of previous interaction for continuation + continuation_token: Token from a `requires_continuation` interaction to resume its long decode extra_headers: Additional headers extra_body: Additional body parameters timeout: Request timeout diff --git a/litellm/interactions/utils.py b/litellm/interactions/utils.py index 3895a85061d..1d7465a99c1 100644 --- a/litellm/interactions/utils.py +++ b/litellm/interactions/utils.py @@ -20,6 +20,7 @@ INTERACTIONS_API_OPTIONAL_PARAMS: Final = { "response_format", "response_mime_type", "previous_interaction_id", + "continuation_token", "agent_config", } diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index 1895a97af7d..50da73cc8b9 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -126,6 +126,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): "response_format", "response_mime_type", "previous_interaction_id", + "continuation_token", ] def validate_environment( @@ -206,6 +207,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): "environment", "response_modalities", "previous_interaction_id", + "continuation_token", ] dropped_keys: Final = ( frozenset({"agent_config"}) @@ -301,7 +303,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): ) -> InteractionsAPIStreamingResponse: """Parse streaming chunk.""" verbose_logger.debug("Google AI Interactions streaming chunk: %s", parsed_chunk) - return InteractionsAPIStreamingResponse(**parsed_chunk) + return InteractionsAPIStreamingResponse.model_validate(parsed_chunk) # GET / DELETE / CANCEL - just build URLs, responses match spec directly diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 148d465099f..08942851af9 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -271,6 +271,10 @@ REQUIRED_ONE_OF_BODY_PARAMS_BY_ROUTE: Final[Mapping[str, tuple[str, str]]] = Map {"acreate_interaction": ("model", "agent"), "acreate_agent": ("id", "name")} ) +BODY_PARAM_ALTERNATIVES_BY_ROUTE: Final[Mapping[str, Mapping[str, tuple[str, ...]]]] = MappingProxyType( + {"acreate_interaction": MappingProxyType({"input": ("continuation_token",)})} +) + JSON_OBJECT_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_JSON_OBJECT_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter( Mapping[str, object] | None @@ -320,8 +324,11 @@ def _find_missing_required_body_param( ) if missing_merge_base_param is not None: return MissingBodyParam(name=missing_merge_base_param, model_deployments_loaded=True) + alternatives: Final = BODY_PARAM_ALTERNATIVES_BY_ROUTE.get(route_type, {}) missing_present_params: Final = tuple( - param for param in REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE.get(route_type, ()) if param not in data + param + for param in REQUIRED_PRESENT_BODY_PARAMS_BY_ROUTE.get(route_type, ()) + if param not in data and not any(alternative in data for alternative in alternatives.get(param, ())) ) if not missing_present_params: return None diff --git a/litellm/types/interactions/generated.py b/litellm/types/interactions/generated.py index c4f7a2d8a7b..4d808742434 100644 --- a/litellm/types/interactions/generated.py +++ b/litellm/types/interactions/generated.py @@ -176,11 +176,16 @@ class Status1(Enum): incomplete = "incomplete" budget_exceeded = "budget_exceeded" queued = "queued" + requires_continuation = "requires_continuation" class InteractionStatusUpdate(LiteLLMBaseModel): interaction_id: str | None = None status: Status1 | None = None + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) event_type: Literal["interaction.status_update"] = "interaction.status_update" event_id: str | None = Field( None, @@ -343,6 +348,7 @@ class Status3(Enum): INCOMPLETE = "incomplete" BUDGET_EXCEEDED = "budget_exceeded" QUEUED = "queued" + REQUIRES_CONTINUATION = "requires_continuation" class ModelOption(RootModel[str]): @@ -604,6 +610,9 @@ class ToolChoice(RootModel[ToolChoiceType | ToolChoiceConfig]): class Usage(LiteLLMBaseModel): total_input_tokens: int | None = Field(None, description="Number of tokens in the prompt (context).") + initial_input_tokens: int | None = Field( + None, description="Number of tokens in the prompt of the first hop of a long decode." + ) input_tokens_by_modality: list[ModalityTokens] | None = Field( None, description="A breakdown of input token usage by modality." ) @@ -625,6 +634,7 @@ class Usage(LiteLLMBaseModel): None, description="A breakdown of tool-use token usage by modality." ) total_reasoning_tokens: int | None = Field(None, description="Number of tokens of thoughts for thinking models.") + total_thought_tokens: int | None = Field(None, description="Number of tokens of thoughts for thinking models.") total_tokens: int | None = Field( None, description="Total token count for the interaction request (prompt + responses + other\ninternal tokens).", @@ -787,6 +797,10 @@ class Interaction(LiteLLMBaseModel): description="The mime type of the response. This is required if response_format is set.", ) previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.") + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) input: str | list[Content] | list[Turn] | Content | None = Field( None, description="The inputs for the interaction." ) @@ -843,6 +857,10 @@ class CreateModelInteractionParams(LiteLLMBaseModel): description="The mime type of the response. This is required if response_format is set.", ) previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.") + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) input: str | list[Content] | list[Turn] | Content = Field(..., description="The inputs for the interaction.") generation_config: GenerationConfig | None = Field( None, @@ -894,6 +912,10 @@ class CreateAgentInteractionParams(LiteLLMBaseModel): description="The mime type of the response. This is required if response_format is set.", ) previous_interaction_id: str | None = Field(None, description="The ID of the previous interaction, if any.") + continuation_token: str | None = Field( + None, + description="Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + ) input: str | list[Content] | list[Turn] | Content = Field(..., description="The inputs for the interaction.") agent_config: DynamicAgentConfig | DeepResearchAgentConfig | None = Field( None, description="Configuration for the agent.", discriminator="type" @@ -1080,6 +1102,7 @@ class InteractionsAPIResponse(BaseLiteLLMOpenAIResponseObject): # New schema field (Api-Revision: 2026-05-20). steps: list[dict[str, Any]] | None = None usage: dict[str, Any] | None = None + continuation_token: str | None = None _hidden_params: dict = PrivateAttr(default_factory=dict) @@ -1126,6 +1149,7 @@ class InteractionsAPIStreamingResponse(BaseLiteLLMOpenAIResponseObject): step: dict[str, Any] | None = None interaction_id: str | None = None interaction: dict[str, Any] | None = None + continuation_token: str | None = None _hidden_params: dict = PrivateAttr(default_factory=dict) diff --git a/tests/integration/providers/test_gemini_interactions_wire.py b/tests/integration/providers/test_gemini_interactions_wire.py index 58a7ccb31c3..ded0718bd2b 100644 --- a/tests/integration/providers/test_gemini_interactions_wire.py +++ b/tests/integration/providers/test_gemini_interactions_wire.py @@ -1,5 +1,4 @@ import json -import uuid from pathlib import Path from typing import Final @@ -147,6 +146,7 @@ def test_interaction_forwards_native_body_to_gemini(gateway: Gateway, prefix: st "agent": None, "updated": None, "outputs": None, + "continuation_token": None, }, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text @@ -179,6 +179,7 @@ def test_interaction_authenticates_google_key_header(gateway: Gateway, prefix: s "agent": None, "updated": None, "outputs": None, + "continuation_token": None, }, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text @@ -218,6 +219,7 @@ def test_agent_interaction_uses_env_gemini_credentials(gateway: Gateway, tmp_pat "agent": "deep-research-pro-preview-12-2025", "updated": None, "outputs": None, + "continuation_token": None, }, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions")], response.text @@ -277,3 +279,78 @@ def test_stream_interaction_ends_without_openai_done_terminator(gateway: Gateway expected: Final = b"".join(_proxy_stream_event(payload, model) for _event_type, payload in _STREAM_EVENTS) assert response.content == expected, response.text assert [(r.method, r.target) for r in wire.drain()] == [("POST", "/v1beta/interactions?alt=sse")] + + +_LONG_DECODE_MODEL: Final = "gemini/gemini-4-argon" +_LONG_DECODE_BODY: Final = { + "input": "prove the prime number theorem", + "generation_config": {"max_output_tokens": 120000, "thinking_level": "high"}, +} + + +def _hop(hop_id: str, status: str, text: str, token: str | None, input_tokens: int, output_tokens: int) -> dict: + return { + "id": hop_id, + "object": "interaction", + "model": "gemini-4-argon", + "status": status, + "created": "2026-11-05T00:00:00Z", + "steps": [{"type": "model_output", "content": [{"type": "text", "text": text}]}], + "usage": { + "total_input_tokens": input_tokens, + "total_output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "initial_input_tokens": 79, + }, + "continuation_token": token, + } + + +_FIRST_HOP: Final = _hop("interaction-hop-1", "requires_continuation", "Lemma 1.", "tok-1", 79, 32768) +_SECOND_HOP: Final = _hop("interaction-hop-2", "completed", " QED.", None, 32847, 1533) + + +def test_interaction_keeps_requires_continuation_status_and_token_for_the_caller(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/v1beta/interactions" + return Reply(body=json.dumps(_FIRST_HOP).encode()) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_LONG_DECODE_MODEL, api_base=wire.url, api_key=_GEMINI_DEPLOYMENT_KEY) + key: Final = scenario.key(models=[model]) + response: Final = gateway.request( + "POST", "/v1beta/interactions", {"model": model, **_LONG_DECODE_BODY}, key=key + ) + assert response.status_code == 200, response.text + sent: Final = [json.loads(r.body) for r in wire.drain()] + + assert sent == [{"model": "gemini-4-argon", **_LONG_DECODE_BODY}], sent + payload: Final = response.json() + assert payload["id"] == "interaction-hop-1", response.text + assert payload["status"] == "requires_continuation", response.text + assert payload["continuation_token"] == "tok-1", response.text + assert payload["usage"]["initial_input_tokens"] == 79, response.text + + +def test_interaction_forwards_a_caller_continuation_token_to_resume_a_long_decode(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/v1beta/interactions" + return Reply(body=json.dumps(_SECOND_HOP).encode()) + + resume: Final = { + "generation_config": _LONG_DECODE_BODY["generation_config"], + "previous_interaction_id": "interaction-hop-1", + "continuation_token": "tok-1", + } + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=_LONG_DECODE_MODEL, api_base=wire.url, api_key=_GEMINI_DEPLOYMENT_KEY) + key: Final = scenario.key(models=[model]) + response: Final = gateway.request("POST", "/v1beta/interactions", {"model": model, **resume}, key=key) + assert response.status_code == 200, response.text + sent: Final = [json.loads(r.body) for r in wire.drain()] + + assert sent == [{"model": "gemini-4-argon", **resume}], sent + payload: Final = response.json() + assert payload["id"] == "interaction-hop-2", response.text + assert payload["status"] == "completed", response.text + assert payload["continuation_token"] is None, response.text diff --git a/tests/unit/interactions/interactions.openapi.json b/tests/unit/interactions/interactions.openapi.json index 3830f5093cb..b67ad0aceaa 100644 --- a/tests/unit/interactions/interactions.openapi.json +++ b/tests/unit/interactions/interactions.openapi.json @@ -756,7 +756,7 @@ { "label": "list", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n res, err := client.Environments.ListEnvironments(ctx, operations.ListEnvironmentsRequest{})\n if err != nil {\n log.Fatal(err)\n }\n\n if res.ListEnvironmentsResponse != nil {\n for _, env := range res.ListEnvironmentsResponse.Environments {\n if env.ID != \"\" {\n fmt.Println(env.ID)\n }\n }\n }\n}\n" } ] }, @@ -834,12 +834,12 @@ { "label": "create", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n req := operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n Sources: []gaos_interactions.Source{\n {\n Type: genai.Ptr(gaos_interactions.SourceTypeInline),\n Target: genai.Ptr(\"main.py\"),\n Content: genai.Ptr(\"print('Hello, World!')\"),\n },\n },\n },\n }\n\n res, err := client.Environments.CreateEnvironment(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" }, { "label": "copy", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_environments \"google.golang.org/genai/interactions/models/environments\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.CreateEnvironment(ctx, operations.CreateEnvironmentRequest{\n Body: gaos_environments.CreateEnvironmentRequest{\n FromEnvironment: genai.Ptr(sourceEnvID),\n },\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n\n }\n}\n" } ] } @@ -967,12 +967,12 @@ { "label": "list_files", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"src\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n fileType := \"\"\n if file.Type != nil {\n fileType = string(*file.Type)\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, fileType, size)\n }\n }\n}\n" }, { "label": "get_file", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.Files.List(ctx, operations.GetEnvironmentFilesRequest{\n Environment: envID,\n Path: \"main.py\",\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.GetEnvironmentFilesResponse != nil {\n for _, file := range res.GetEnvironmentFilesResponse.Files {\n name := \"\"\n if file.Name != nil {\n name = *file.Name\n }\n var size int64\n if file.SizeBytes != nil {\n size = *file.SizeBytes\n }\n fmt.Println(name, size)\n }\n }\n}\n" } ] } @@ -1036,7 +1036,7 @@ { "label": "delete", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n _, err = client.Environments.DeleteEnvironment(ctx, operations.DeleteEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n fmt.Println(\"Environment deleted successfully.\")\n}\n" } ] }, @@ -1093,7 +1093,7 @@ { "label": "get", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client := interactions.New()\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n\n res, err := client.Environments.GetEnvironment(ctx, operations.GetEnvironmentRequest{\n ID: envID,\n })\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Environment != nil && res.Environment.ID != \"\" {\n fmt.Println(res.Environment.ID)\n }\n}\n" } ] } @@ -1171,7 +1171,7 @@ "deep_research": { "summary": "Deep Research", "value": { - "agent": "deep-research-pro-preview-12-2025", + "agent": "deep-research-preview-04-2026", "created": "2025-11-26T12:22:47Z", "id": "v1_ChdPU0F4YWFtNkFwS2kxZThQZ05lbXdROBIXT1NBeGFhbTZBcEtpMWU4UGdOZW13UTg", "object": "interaction", @@ -1472,7 +1472,7 @@ { "label": "deep_research", "lang": "sh", - "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-pro-preview-12-2025\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n" + "source": "curl -X POST https://generativelanguage.googleapis.com/v1beta/interactions \\\n -H \"x-goog-api-key: $GEMINI_API_KEY\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"agent\": \"deep-research-preview-04-2026\",\n \"input\": \"Find a cure to cancer\",\n \"background\": true\n }'\n" }, { "label": "antigravity", @@ -1517,7 +1517,7 @@ { "label": "deep_research", "lang": "python", - "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-pro-preview-12-2025\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n" + "source": "from google import genai\n\nclient = genai.Client()\ninteraction = client.interactions.create(\n agent=\"deep-research-preview-04-2026\",\n input=\"find a cure to cancer\",\n background=True,\n)\nprint(interaction.status)\n" }, { "label": "antigravity", @@ -1562,7 +1562,7 @@ { "label": "deep_research", "lang": "javascript", - "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-pro-preview-12-2025',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n" + "source": "import {GoogleGenAI} from '@google/genai';\n\nconst ai = new GoogleGenAI({});\nconst interaction = await ai.interactions.create({\n agent: 'deep-research-preview-04-2026',\n input: 'find a cure to cancer',\n background: true,\n});\nconsole.log(interaction.status);\n" }, { "label": "antigravity", @@ -1607,7 +1607,7 @@ { "label": "deep_research", "lang": "java", - "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-pro-preview-12-2025\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" + "source": "import com.google.genai.gaos.models.interactions.AgentOption;\nimport com.google.genai.Client;\nimport com.google.genai.gaos.models.interactions.CreateAgentInteraction;\nimport com.google.genai.gaos.models.operations.CreateInteractionRequestBody;\nimport com.google.genai.gaos.models.operations.CreateInteractionResponse;\nimport com.google.genai.gaos.models.interactions.Interaction;\nimport com.google.genai.gaos.models.interactions.InteractionStatus;\nimport com.google.genai.gaos.models.interactions.InteractionsInput;\n\nClient client = new Client();\nCreateAgentInteraction params =\n CreateAgentInteraction.builder()\n .agent(AgentOption.of(\"deep-research-preview-04-2026\"))\n .input(InteractionsInput.of(\"find a cure to cancer\"))\n .background(true)\n .build();\nCreateInteractionResponse response =\n client.interactions.create(CreateInteractionRequestBody.of(params));\nInteraction interaction =\n response.interaction().orElseThrow(() -> new RuntimeException(\"No interaction returned\"));\nSystem.out.println(interaction.status().map(InteractionStatus::value).orElse(\"\"));\n" }, { "label": "antigravity", @@ -1652,7 +1652,7 @@ { "label": "deep_research", "lang": "go", - "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-pro-preview-12-2025\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n" + "source": "package main\n\nimport (\n \"context\"\n \"fmt\"\n \"log\"\n \"google.golang.org/genai\"\n gaos_interactions \"google.golang.org/genai/interactions/models/interactions\"\n \"google.golang.org/genai/interactions/models/operations\"\n)\n\nfunc main() {\n ctx := context.Background()\n client, err := genai.NewClient(ctx, nil)\n if err != nil {\n log.Fatal(err)\n }\n\n body := operations.NewCreateInteractionRequestBody(gaos_interactions.CreateAgentInteraction{\n Agent: gaos_interactions.AgentOption(\"deep-research-preview-04-2026\"),\n Input: genai.Ptr(gaos_interactions.NewInteractionsInput(\"find a cure to cancer\")),\n Background: genai.Ptr(true),\n })\n\n req := operations.CreateInteractionRequest{\n Body: body,\n }\n\n res, err := client.Interactions.Create(ctx, req)\n if err != nil {\n log.Fatal(err)\n }\n\n if res.Interaction != nil {\n fmt.Println(res.Interaction.Status)\n }\n}\n" }, { "label": "antigravity", @@ -3041,6 +3041,22 @@ "description": "An agent definition for the CreateAgent API.\nThis message is the target for annotation-parser-based JSON parsing.\nNew format:\n {\n \"id\": \"customer-sentinel\",\n \"base_agent\": \"\",\n \"system_instruction\": \"...\",\n \"base_environment\": { \"type\": \"remote\", \"sources\": [...] },\n \"tools\": [ {\"type\": \"code_execution\"} ]\n }", "type": "object", "properties": { + "id": { + "description": "The unique identifier for the agent.", + "type": "string" + }, + "base_agent": { + "description": "The base agent to extend.", + "type": "string" + }, + "system_instruction": { + "description": "System instruction for the agent.", + "type": "string" + }, + "description": { + "description": "Agent description for developers to quickly read and understand.", + "type": "string" + }, "agent_config": { "description": "Configuration parameters for the agent.", "oneOf": [ @@ -3060,22 +3076,6 @@ } ] }, - "id": { - "description": "The unique identifier for the agent.", - "type": "string" - }, - "base_agent": { - "description": "The base agent to extend.", - "type": "string" - }, - "system_instruction": { - "description": "System instruction for the agent.", - "type": "string" - }, - "description": { - "description": "Agent description for developers to quickly read and understand.", - "type": "string" - }, "tools": { "description": "The tools available to the agent.", "type": "array", @@ -3091,13 +3091,11 @@ "description": "The agent to interact with.", "type": "string", "enum": [ - "deep-research-pro-preview-12-2025", "deep-research-preview-04-2026", "deep-research-max-preview-04-2026", "antigravity-preview-05-2026" ], "x-speakeasy-enum-descriptions": [ - "Gemini Deep Research Agent", "Gemini Deep Research Agent", "Gemini Deep Research Max Agent", "Use the Antigravity managed agent to perform multi-step tasks that require reasoning, file operations, and tool use." @@ -3330,14 +3328,6 @@ "type": { "const": "audio" }, - "data": { - "type": "string", - "format": "byte", - "x-speakeasy-base64-input-mode": "file" - }, - "uri": { - "type": "string" - }, "mime_type": { "type": "string", "enum": [ @@ -3371,6 +3361,14 @@ "WEBM audio format" ] }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, "rate": { "description": "Deprecated. Use sample_rate instead. The value is ignored.", "type": "integer", @@ -3817,32 +3815,6 @@ } ] }, - "response_format": { - "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", - "oneOf": [ - { - "$ref": "#/components/schemas/ResponseFormat" - }, - { - "title": "ResponseFormatList", - "type": "array", - "items": { - "$ref": "#/components/schemas/ResponseFormat" - } - } - ] - }, - "environment": { - "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", - "oneOf": [ - { - "$ref": "#/components/schemas/EnvironmentConfig" - }, - { - "type": "string" - } - ] - }, "id": { "description": "Required. Output only. A unique identifier for the interaction completion.", "type": "string", @@ -3858,7 +3830,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "readOnly": true, "x-google-enum-deprecated": [ @@ -3868,7 +3841,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -3877,7 +3851,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "created": { @@ -3909,6 +3884,21 @@ "$ref": "#/components/schemas/ResponseModality" } }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, "response_mime_type": { "description": "The mime type of the response. This is required if response_format is set.", "type": "string", @@ -3918,6 +3908,17 @@ "description": "The ID of the previous interaction, if any.", "type": "string" }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, "environment_id": { "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", "type": "string", @@ -3945,6 +3946,12 @@ "type": "string" } }, + "continuation_token": { + "description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, "stream": { "description": "Input only. Whether the interaction will be streamed.", "type": "boolean", @@ -4018,32 +4025,6 @@ "$ref": "#/components/schemas/GenerationConfig", "writeOnly": true }, - "response_format": { - "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", - "oneOf": [ - { - "$ref": "#/components/schemas/ResponseFormat" - }, - { - "title": "ResponseFormatList", - "type": "array", - "items": { - "$ref": "#/components/schemas/ResponseFormat" - } - } - ] - }, - "environment": { - "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", - "oneOf": [ - { - "$ref": "#/components/schemas/EnvironmentConfig" - }, - { - "type": "string" - } - ] - }, "id": { "description": "Required. Output only. A unique identifier for the interaction completion.", "type": "string", @@ -4059,7 +4040,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "readOnly": true, "x-google-enum-deprecated": [ @@ -4069,7 +4051,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -4078,7 +4061,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "created": { @@ -4110,6 +4094,21 @@ "$ref": "#/components/schemas/ResponseModality" } }, + "response_format": { + "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", + "oneOf": [ + { + "$ref": "#/components/schemas/ResponseFormat" + }, + { + "title": "ResponseFormatList", + "type": "array", + "items": { + "$ref": "#/components/schemas/ResponseFormat" + } + } + ] + }, "response_mime_type": { "description": "The mime type of the response. This is required if response_format is set.", "type": "string", @@ -4119,6 +4118,17 @@ "description": "The ID of the previous interaction, if any.", "type": "string" }, + "environment": { + "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentConfig" + }, + { + "type": "string" + } + ] + }, "environment_id": { "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", "type": "string", @@ -4146,6 +4156,12 @@ "type": "string" } }, + "continuation_token": { + "description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, "stream": { "description": "Input only. Whether the interaction will be streamed.", "type": "boolean", @@ -4348,11 +4364,31 @@ "type": "string", "enum": [ "application/pdf", - "text/csv" + "text/csv", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.oasis.opendocument.text", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet" ], "x-google-enum-descriptions": [ "PDF document format", - "CSV document format" + "CSV document format", + "PPTX presentation format", + "PPT presentation format", + "ODP presentation format", + "DOCX document format", + "DOC document format", + "DOTX document template format", + "ODT document format", + "XLSX spreadsheet format", + "XLS spreadsheet format", + "ODS spreadsheet format" ] } }, @@ -4379,6 +4415,37 @@ "type": { "const": "document" }, + "mime_type": { + "type": "string", + "enum": [ + "application/pdf", + "text/csv", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.ms-powerpoint", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "application/vnd.oasis.opendocument.text", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet" + ], + "x-google-enum-descriptions": [ + "PDF document format", + "CSV document format", + "PPTX presentation format", + "PPT presentation format", + "ODP presentation format", + "DOCX document format", + "DOC document format", + "DOTX document template format", + "ODT document format", + "XLSX spreadsheet format", + "XLS spreadsheet format", + "ODS spreadsheet format" + ] + }, "data": { "type": "string", "format": "byte", @@ -4386,17 +4453,6 @@ }, "uri": { "type": "string" - }, - "mime_type": { - "type": "string", - "enum": [ - "application/pdf", - "text/csv" - ], - "x-google-enum-descriptions": [ - "PDF document format", - "CSV document format" - ] } }, "required": [ @@ -4474,23 +4530,6 @@ "type": "string", "readOnly": true }, - "network": { - "description": "Network configuration for the environment.", - "oneOf": [ - { - "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" - }, - { - "type": "string", - "enum": [ - "disabled" - ], - "x-google-enum-descriptions": [ - "All network egress is blocked." - ] - } - ] - }, "sources": { "description": "Sources to be mounted into the environment.", "type": "array", @@ -4526,6 +4565,23 @@ "" ] }, + "network": { + "description": "Network configuration for the environment.", + "oneOf": [ + { + "$ref": "#/components/schemas/EnvironmentNetworkEgressAllowlist" + }, + { + "type": "string", + "enum": [ + "disabled" + ], + "x-google-enum-descriptions": [ + "All network egress is blocked." + ] + } + ] + }, "file_count": { "description": "Output only. The number of files in the environment, output only.", "type": "string", @@ -4551,6 +4607,12 @@ "type": { "const": "remote" }, + "sources": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Source" + } + }, "network": { "description": "Network configuration for the environment.", "oneOf": [ @@ -4568,12 +4630,6 @@ } ] }, - "sources": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Source" - } - }, "environment_id": { "description": "Optional. The environment ID for the interaction. If specified, the request will\nupdate the existing environment instead of creating a new one.", "type": "string" @@ -5347,30 +5403,6 @@ "description": "Configuration parameters for model interactions.", "type": "object", "properties": { - "tool_choice": { - "description": "The tool choice configuration.", - "oneOf": [ - { - "$ref": "#/components/schemas/ToolChoiceConfig" - }, - { - "type": "string", - "enum": [ - "auto", - "any", - "none", - "validated" - ], - "x-google-enum-descriptions": [ - "Auto tool choice.", - "Any tool choice.", - "No tool choice.", - "Validated tool choice." - ], - "x-speakeasy-model-namespace": "interactions" - } - ] - }, "temperature": { "description": "Controls the randomness of the output.", "type": "number", @@ -5395,6 +5427,30 @@ "type": "string" } }, + "tool_choice": { + "description": "The tool choice configuration.", + "oneOf": [ + { + "$ref": "#/components/schemas/ToolChoiceConfig" + }, + { + "type": "string", + "enum": [ + "auto", + "any", + "none", + "validated" + ], + "x-google-enum-descriptions": [ + "Auto tool choice.", + "Any tool choice.", + "No tool choice.", + "Validated tool choice." + ], + "x-speakeasy-model-namespace": "interactions" + } + ] + }, "thinking_level": { "description": "The level of thought tokens that the model should generate.", "$ref": "#/components/schemas/ThinkingLevel" @@ -6211,14 +6267,6 @@ "type": { "const": "image" }, - "data": { - "type": "string", - "format": "byte", - "x-speakeasy-base64-input-mode": "file" - }, - "uri": { - "type": "string" - }, "mime_type": { "type": "string", "enum": [ @@ -6242,6 +6290,14 @@ "TIFF image format" ] }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, "resolution": { "description": "The resolution of the media.", "$ref": "#/components/schemas/MediaResolution" @@ -6385,7 +6441,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "readOnly": true, "x-google-enum-deprecated": [ @@ -6395,7 +6452,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -6404,7 +6462,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "created": { @@ -6434,9 +6493,33 @@ "$ref": "#/components/schemas/GenerationConfig", "writeOnly": true }, + "system_instruction": { + "description": "System instruction for the interaction.", + "type": "string" + }, + "tools": { + "description": "A list of tool declarations the model may call during interaction.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Tool" + } + }, "input": { "$ref": "#/components/schemas/InteractionsInput" }, + "usage": { + "description": "Output only. Statistics on the interaction request's token usage.", + "$ref": "#/components/schemas/Usage", + "readOnly": true + }, + "response_modalities": { + "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", + "type": "array", + "deprecated": true, + "items": { + "$ref": "#/components/schemas/ResponseModality" + } + }, "response_format": { "description": "Enforces that the generated response is a JSON object that complies with\nthe JSON schema specified in this field.", "oneOf": [ @@ -6452,6 +6535,15 @@ } ] }, + "response_mime_type": { + "description": "The mime type of the response. This is required if response_format is set.", + "type": "string", + "deprecated": true + }, + "previous_interaction_id": { + "description": "The ID of the previous interaction, if any.", + "type": "string" + }, "environment": { "description": "The environment configuration for the interaction. Can be an object\nspecifying remote environment sources or a string referencing an existing\nenvironment ID.", "oneOf": [ @@ -6463,39 +6555,6 @@ } ] }, - "system_instruction": { - "description": "System instruction for the interaction.", - "type": "string" - }, - "tools": { - "description": "A list of tool declarations the model may call during interaction.", - "type": "array", - "items": { - "$ref": "#/components/schemas/Tool" - } - }, - "usage": { - "description": "Output only. Statistics on the interaction request's token usage.", - "$ref": "#/components/schemas/Usage", - "readOnly": true - }, - "response_modalities": { - "description": "The requested modalities of the response (TEXT, IMAGE, AUDIO).", - "type": "array", - "deprecated": true, - "items": { - "$ref": "#/components/schemas/ResponseModality" - } - }, - "response_mime_type": { - "description": "The mime type of the response. This is required if response_format is set.", - "type": "string", - "deprecated": true - }, - "previous_interaction_id": { - "description": "The ID of the previous interaction, if any.", - "type": "string" - }, "environment_id": { "description": "Output only. The environment ID for the interaction. Only populated if environment\nconfig is set in the request.", "type": "string", @@ -6531,6 +6590,12 @@ }, "readOnly": true }, + "continuation_token": { + "description": "Opaque token to resume a long decode. Output: set when status is\nREQUIRES_CONTINUATION and decoding can be resumed. Input: pass the latest\ntoken back unchanged in CreateInteraction to continue decoding.", + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, "stream": { "description": "Input only. Whether the interaction will be streamed.", "type": "boolean", @@ -6813,16 +6878,16 @@ }, "readOnly": true }, - "agent": { - "description": "The agent to interact with.", - "type": "string" - }, "continuation_token": { "description": "Opaque token to resume a long decode.\nOutput: set when status is INCOMPLETE and decoding can be resumed.\nInput: pass the latest token back unchanged in CreateInteraction to\ncontinue decoding.", "type": "string", "format": "byte", "x-speakeasy-base64-input-mode": "file" }, + "agent": { + "description": "The agent to interact with.", + "type": "string" + }, "object": { "description": "Output only. The resource type.", "type": "string", @@ -6863,7 +6928,8 @@ "failed", "cancelled", "incomplete", - "budget_exceeded" + "budget_exceeded", + "requires_continuation" ], "x-google-enum-deprecated": [ false, @@ -6872,7 +6938,8 @@ false, false, false, - true + true, + false ], "x-google-enum-descriptions": [ "The interaction is in progress.", @@ -6881,7 +6948,8 @@ "The interaction failed.", "The interaction was cancelled.", "The interaction is completed, but contains incomplete results (e.g.\nhitting max_tokens).", - "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11)." + "Deprecated: Token and execution budget exhaustion returns INCOMPLETE\n(11).", + "The interaction requires at least one more request for it to be\ncompleted. The next request should set the continuation_token to the one\nreturned by this request." ] }, "event_type": { @@ -7076,14 +7144,14 @@ "type": { "const": "mcp_server" }, - "url": { - "description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"", - "type": "string" - }, "name": { "description": "The name of the MCPServer.", "type": "string" }, + "url": { + "description": "The full URL for the MCPServer endpoint.\nExample: \"https://api.example.com/mcp\"", + "type": "string" + }, "headers": { "description": "Optional: Fields for authentication headers, timeouts, etc., if needed.", "type": "object", @@ -7362,7 +7430,7 @@ }, "ModelOption": { "title": "Model", - "description": "The model that will complete your prompt.\\n\\nSee [models](https://ai.google.dev/gemini-api/docs/models) for additional details.", + "description": "The model that will complete your prompt.\\n\\nSee [models](https://aistudio.google.com/docs/models) for additional details.", "type": "string", "enum": [ "gemma-4-26b-a4b-it", @@ -8695,6 +8763,20 @@ "description": "Configuration for speech recognition (transcription).", "type": "object", "properties": { + "language_codes": { + "description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.", + "type": "array", + "items": { + "type": "string" + } + }, + "custom_vocabulary": { + "description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.", + "type": "array", + "items": { + "type": "string" + } + }, "mode": { "description": "Discriminated transcription mode options or enum.", "oneOf": [ @@ -8714,20 +8796,6 @@ } ] }, - "language_codes": { - "description": "Optional. BCP-47 language codes providing hints about the languages present in the\naudio. If omitted or empty, defaults to automatic language detection.", - "type": "array", - "items": { - "type": "string" - } - }, - "custom_vocabulary": { - "description": "Optional. A list of custom vocabulary phrases to bias the speech recognition model\ntoward recognizing specific terms.", - "type": "array", - "items": { - "type": "string" - } - }, "timestamp_granularities": { "description": "Optional. The granularity of timestamps to include in the transcription output.\nSupported values: \"word\". If empty, no timestamps are generated.", "type": "array", @@ -9332,6 +9400,11 @@ "type": "integer", "format": "int32" }, + "initial_input_tokens": { + "description": "Number of input tokens used in the first turn of this interaction.", + "type": "integer", + "format": "int32" + }, "input_tokens_by_modality": { "description": "A breakdown of input token usage by modality.", "type": "array", @@ -9497,6 +9570,10 @@ "description": "The URI of the video.", "type": "string" }, + "resolution": { + "description": "The resolution of the media.", + "$ref": "#/components/schemas/MediaResolution" + }, "processing": { "description": "How the model processes this video for understanding.", "oneOf": [ @@ -9516,10 +9593,6 @@ } ] }, - "resolution": { - "description": "The resolution of the media.", - "$ref": "#/components/schemas/MediaResolution" - }, "name": { "description": "A user-defined name for this content block. Can be referenced by the model\nin the final response.", "type": "string" @@ -9573,14 +9646,6 @@ "type": { "const": "video" }, - "data": { - "type": "string", - "format": "byte", - "x-speakeasy-base64-input-mode": "file" - }, - "uri": { - "type": "string" - }, "mime_type": { "type": "string", "enum": [ @@ -9608,6 +9673,14 @@ "JPEG 2000 video format" ] }, + "data": { + "type": "string", + "format": "byte", + "x-speakeasy-base64-input-mode": "file" + }, + "uri": { + "type": "string" + }, "resolution": { "description": "The resolution of the media.", "$ref": "#/components/schemas/MediaResolution" diff --git a/tests/unit/interactions/test_main.py b/tests/unit/interactions/test_main.py index 5af5dd7d82a..cd32eb11485 100644 --- a/tests/unit/interactions/test_main.py +++ b/tests/unit/interactions/test_main.py @@ -559,28 +559,34 @@ class TestInteractionsAcreateOffline: @pytest.mark.usefixtures("fake_provider_credentials") @pytest.mark.asyncio - async def test_acreate_gemini_keeps_continuation_token_out_of_the_request(self, respx_mock: MockRouter) -> None: + async def test_acreate_gemini_sends_a_caller_continuation_token_to_resume_a_decode( + self, respx_mock: MockRouter + ) -> None: route: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( return_value=httpx.Response( 200, json={ "id": "interaction-offline", "object": "interaction", - "model": "gemini-3.8-flash", + "model": "gemini-4-argon", "status": "completed", "steps": [{"type": "model_output", "content": [{"type": "text", "text": "Hi"}]}], - "usage": {"input_tokens": 6, "output_tokens": 3}, + "usage": {"total_input_tokens": 6, "total_output_tokens": 3}, }, ) ) response: Final = await interactions.acreate( - model="gemini/gemini-3.8-flash", - input="Prove it", + model="gemini/gemini-4-argon", + previous_interaction_id="interaction-hop-1", continuation_token="dG9rLTE=", api_key="gemini-offline", ) - # The Interactions API answered 400 "Unknown parameter 'continuation_token'." on 2026-10-13 - assert json.loads(route.calls.last.request.content) == {"model": "gemini-3.8-flash", "input": "Prove it"} + # Google's spec (2026-11-05) takes continuation_token on CreateModelInteractionParams; a 2026-10-13 400 is gone + assert json.loads(route.calls.last.request.content) == { + "model": "gemini-4-argon", + "previous_interaction_id": "interaction-hop-1", + "continuation_token": "dG9rLTE=", + } assert response.status == "completed" @pytest.mark.usefixtures("fake_provider_credentials") @@ -746,9 +752,7 @@ def test_create_with_system_instruction_for_google_and_responses( def test_create_streaming_returns_interaction_events( respx_mock: MockRouter, ) -> None: - route: Final = respx_mock.post( - "https://generativelanguage.googleapis.com/v1beta/interactions?alt=sse" - ).mock( + route: Final = respx_mock.post("https://generativelanguage.googleapis.com/v1beta/interactions?alt=sse").mock( return_value=httpx.Response( 200, headers={"content-type": "text/event-stream"}, @@ -856,3 +860,65 @@ def test_create_streaming_hands_completed_interaction_to_success_callback( ) assert recorder.logged == (_GEMINI_COMPLETED_INTERACTION,) + + +_GEMINI_URL: Final = "https://generativelanguage.googleapis.com/v1beta/interactions" +_LONG_DECODE_REQUEST: Final = { + "model": "gemini/gemini-4-argon", + "input": "Prove the prime number theorem.", + "generation_config": {"max_output_tokens": 120000, "thinking_level": "high"}, +} +_FIRST_HOP: Final = { + "id": "hop-1", + "object": "interaction", + "model": "gemini-4-argon", + "status": "requires_continuation", + "steps": [{"type": "model_output", "content": [{"type": "text", "text": "Lemma 1."}]}], + "usage": { + "total_input_tokens": 79, + "total_output_tokens": 32668, + "total_thought_tokens": 100, + "total_tokens": 32847, + "initial_input_tokens": 79, + }, + "continuation_token": "tok-1", +} + + +@pytest.mark.usefixtures("fake_provider_credentials") +def test_create_keeps_requires_continuation_status_and_token_for_the_caller(respx_mock: MockRouter) -> None: + route: Final = respx_mock.post(_GEMINI_URL).mock(return_value=httpx.Response(200, json=_FIRST_HOP)) + + response: Final = interactions.create(**_LONG_DECODE_REQUEST, api_key="gemini-offline") + + assert route.call_count == 1 + assert response.status == "requires_continuation" + assert response.continuation_token == "tok-1" + assert response.usage == _FIRST_HOP["usage"] + + +@pytest.mark.usefixtures("fake_provider_credentials") +def test_create_sends_a_caller_continuation_token_to_resume_a_long_decode(respx_mock: MockRouter) -> None: + route: Final = respx_mock.post(_GEMINI_URL).mock( + return_value=httpx.Response( + 200, json={**_FIRST_HOP, "id": "hop-2", "status": "completed", "continuation_token": None} + ) + ) + + response: Final = interactions.create( + model="gemini/gemini-4-argon", + previous_interaction_id="hop-1", + continuation_token="tok-1", + generation_config=_LONG_DECODE_REQUEST["generation_config"], + api_key="gemini-offline", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent == { + "model": "gemini-4-argon", + "previous_interaction_id": "hop-1", + "continuation_token": "tok-1", + "generation_config": _LONG_DECODE_REQUEST["generation_config"], + }, sent + assert response.status == "completed" + assert response.continuation_token is None diff --git a/tests/unit/interactions/test_openapi_compliance.py b/tests/unit/interactions/test_openapi_compliance.py index b6cc92e0adc..71db2cd24e3 100644 --- a/tests/unit/interactions/test_openapi_compliance.py +++ b/tests/unit/interactions/test_openapi_compliance.py @@ -2,7 +2,7 @@ OpenAPI compliance tests for Google Interactions API. Validates that our SDK requests/responses match the OpenAPI spec snapshot in -interactions.openapi.json, taken from https://ai.google.dev/static/api/interactions.openapi.json +interactions.openapi.json, taken from https://aistudio.google.com/api-reference/interactions.openapi.json (litellm/types/interactions/README.md says how to refresh it). The scheduled interactions-spec-canary workflow runs the same tests against the live spec by passing --interactions-spec-url, so a change Google makes shows up there, not in every PR. @@ -23,7 +23,7 @@ import pytest from openapi_core import OpenAPI from pydantic import BaseModel, TypeAdapter -OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json" +OPENAPI_SPEC_URL = "https://aistudio.google.com/api-reference/interactions.openapi.json" OPENAPI_SPEC_SNAPSHOT: Final = Path(__file__).with_name("interactions.openapi.json") SPEC_URL_OPTION: Final = TypeAdapter(str | None) @@ -47,10 +47,10 @@ def _requested_spec_url(request: pytest.FixtureRequest) -> str | None: def _load_openapi_spec(live_url: str | None) -> LoadedSpec: if live_url is None: return LoadedSpec(source=str(OPENAPI_SPEC_SNAPSHOT), spec=json.loads(OPENAPI_SPEC_SNAPSHOT.read_text())) - with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0) as client: + with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0, follow_redirects=True) as client: response: Final = client.get(live_url) response.raise_for_status() - return LoadedSpec(source=str(response.url), spec=response.json()) + return LoadedSpec(source=live_url, spec=response.json()) def _model_create_request_schema(spec_dict: Dict[str, Any]) -> Dict[str, Any]: diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 0c7292674a6..892c891f20c 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -1939,6 +1939,7 @@ async def test_callback_handles_every_status_the_interactions_api_can_return(): assert released == { "completed", "requires_action", + "requires_continuation", "failed", "cancelled", "incomplete", diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 71f816ea183..cb903fb134a 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1522,6 +1522,37 @@ def test_raise_if_required_body_param_missing_rejects_missing_param(route_type, assert exc_info.value.message == f"{route}: Missing required parameter: '{param}'." +@pytest.mark.parametrize( + "data", + [ + {"model": "gemini-4-argon", "input": "prove the prime number theorem"}, + {"model": "gemini-4-argon", "previous_interaction_id": "v1_abc", "continuation_token": "tok-1"}, + {"model": "gemini-4-argon", "input": "prove it again", "continuation_token": "tok-1"}, + ], +) +def test_create_interaction_accepts_input_or_a_continuation_token(data: dict[str, object]) -> None: + from litellm.proxy.route_llm_request import raise_if_required_body_param_missing + + assert raise_if_required_body_param_missing(route_type="acreate_interaction", data=data, llm_router=None) is None + + +def test_create_interaction_without_input_or_continuation_token_names_input() -> None: + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing( + route_type="acreate_interaction", + data={"model": "gemini-4-argon", "previous_interaction_id": "v1_abc"}, + llm_router=None, + ) + + assert exc_info.value.param == "input" + assert exc_info.value.message == "/interactions: Missing required parameter: 'input'." + + @pytest.mark.parametrize( "data, param", [ From 41cbbfeb8d31ff05028a9e5a8bc6c9e0385d2c81 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Thu, 5 Nov 2026 15:46:44 -0800 Subject: [PATCH 07/34] fix(router): keep the request's metadata on rust core fallback hop kwargs (#50263) The fallback loggers got the walk's own metadata in place of the request's, so prometheus fallback counters lost the caller's key and team labels. Lay the walk's stamps over a copy of the request's bucket, as run_async_fallback did --- litellm/router_rust/invoke/walk.py | 12 +++++-- tests/unit/router_rust/test_walk.py | 49 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 tests/unit/router_rust/test_walk.py diff --git a/litellm/router_rust/invoke/walk.py b/litellm/router_rust/invoke/walk.py index 2c1481f068a..17e20119f43 100644 --- a/litellm/router_rust/invoke/walk.py +++ b/litellm/router_rust/invoke/walk.py @@ -264,10 +264,18 @@ class WalkMetadata: def hop_kwargs(self, event: Mapping[str, object]) -> dict[str, object]: # mutable-ok: a fresh dict for the loggers """The kwargs the level held after a hop (``kwargs["model"] = target``), as the - fallback loggers see them.""" + fallback loggers see them: the request's own metadata (the proxy's ``user_api_key_*`` + labels) with the level's walk stamps laid over it, as ``run_async_fallback``'s + ``_hop_metadata`` copies it.""" scope: Final = self._scope(event.get("scope")) model: Final = scope.metadata.get("model_group", event.get("to")) - return {**self._request, **scope.kwargs, "model": model, self._field: dict(scope.metadata)} + request_bucket: Final = as_str_mapping(self._request.get(self._field)) or {} + return { + **self._request, + **scope.kwargs, + "model": model, + self._field: {**request_bucket, **scope.metadata}, + } def log_retry(self, event: Mapping[str, object], error: BaseException | None) -> None: """A ``retry_logged`` event: the failure joins the scope's ``previous_models``.""" diff --git a/tests/unit/router_rust/test_walk.py b/tests/unit/router_rust/test_walk.py new file mode 100644 index 00000000000..2f2febea3eb --- /dev/null +++ b/tests/unit/router_rust/test_walk.py @@ -0,0 +1,49 @@ +import pytest + +from litellm.router_rust.invoke.walk import WalkMetadata + + +def _no_base_model(_model_id: str) -> str | None: + return None + + +@pytest.mark.parametrize("field", ["metadata", "litellm_metadata"]) +def test_hop_kwargs_keep_the_requests_key_labels_under_the_walk_stamps(field: str) -> None: + """The fallback loggers (prometheus' fallback counters) read the caller's key and team off + the hop's metadata, so a hop keeps the request's bucket and lays its own stamps over it.""" + request = { + "model": "primary", + field: { + "user_api_key_hash": "hashed-key", + "user_api_key_alias": "key-alias", + "user_api_key_team_id": "team-id", + "user_api_key_team_alias": "team-alias", + "attempted_fallbacks": "stale", + }, + } + walk = WalkMetadata(request, field, _no_base_model) + walk.hop({"parent": 0, "scope": 1, "from": "primary", "to": "fallback", "depth": 1, "max_fallbacks": 5}) + + kwargs = walk.hop_kwargs({"scope": 1, "to": "fallback"}) + + assert kwargs["model"] == "fallback" + metadata = kwargs[field] + assert isinstance(metadata, dict) + assert metadata["user_api_key_hash"] == "hashed-key" + assert metadata["user_api_key_alias"] == "key-alias" + assert metadata["user_api_key_team_id"] == "team-id" + assert metadata["user_api_key_team_alias"] == "team-alias" + assert metadata["model_group"] == "fallback" + assert metadata["original_model_group"] == "primary" + assert metadata["attempted_fallbacks"] == 1 + assert request[field]["attempted_fallbacks"] == "stale" + assert "model_group" not in request[field] + + +def test_hop_kwargs_without_request_metadata_hold_the_walk_stamps() -> None: + walk = WalkMetadata({"model": "primary"}, "metadata", _no_base_model) + walk.hop({"parent": 0, "scope": 1, "from": "primary", "to": "fallback", "depth": 1, "max_fallbacks": 5}) + + metadata = walk.hop_kwargs({"scope": 1, "to": "fallback"})["metadata"] + + assert metadata == {"original_model_group": "primary", "model_group": "fallback", "attempted_fallbacks": 1} From 8592298e5ec9ea724eea31f1122edfae61e42975 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Thu, 5 Nov 2026 15:46:46 -0800 Subject: [PATCH 08/34] fix(router): retry a team wildcard under the provider-prefixed name (#50264) A bare model name that no team pattern matches as spelled is now tried again as provider/model against the caller's team pattern routers, as PatternMatchRouter.get_pattern did in the Python core, with the routed deployment's model filled from the prefixed name. --- litellm-rust/crates/router-core/src/lookup.rs | 4 + .../crates/router-core/src/resolve.rs | 80 ++++++++++++++++--- litellm-rust/crates/router-core/src/select.rs | 8 +- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/litellm-rust/crates/router-core/src/lookup.rs b/litellm-rust/crates/router-core/src/lookup.rs index 39c235d6f7b..8fef0ba222f 100644 --- a/litellm-rust/crates/router-core/src/lookup.rs +++ b/litellm-rust/crates/router-core/src/lookup.rs @@ -433,6 +433,10 @@ impl<'a> Lookup<'a> { .collect() } + pub fn has_team_wildcards(&self, team_id: &str) -> bool { + self.host.team_patterns.iter().any(|(t, _)| t == team_id) + } + /// The deployments `team_id`'s own wildcard pattern routes `name` to /// (`_core_team_pattern_routers`, keyed by `team_public_model_name`). pub fn team_wildcard(&self, name: &str, team_id: &str) -> Vec> { diff --git a/litellm-rust/crates/router-core/src/resolve.rs b/litellm-rust/crates/router-core/src/resolve.rs index 9075fc36c53..d897716b1e5 100644 --- a/litellm-rust/crates/router-core/src/resolve.rs +++ b/litellm-rust/crates/router-core/src/resolve.rs @@ -223,14 +223,20 @@ pub(crate) fn wildcard_name<'a>( registry: &'a Registry, name: &'a str, specific_deployment: bool, + team_id: Option<&str>, ) -> Option<&'a str> { let model = registry.alias_of(name).unwrap_or(name); - (registry.has_wildcards() + let lookup = crate::lookup::Lookup::new(registry); + let team_patterns = team_id.is_some_and(|team| lookup.has_team_wildcards(team)); + let team_misses = || team_id.is_none_or(|team| lookup.team_wildcard(model, team).is_empty()); + let global_misses = || registry.has_wildcards() && registry.wildcard_for(model).is_none(); + ((team_patterns || registry.has_wildcards()) && !specific_deployment && !registry.has_name(model) && (registry.has_name(name) || registry.deployment(name).is_none()) && registry.routing_group_named(model).is_none() - && registry.wildcard_for(model).is_none()) + && team_misses() + && (team_patterns || global_misses())) .then_some(model) } @@ -339,15 +345,21 @@ pub fn resolve_request( let kept = reserved(model, deployments, caller, now)?; return resolved(ResolveKind::Group, model, kept); } - // `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one. - let team_owned = team_id - .map(|team| crate::lookup::Lookup::new(registry).team_wildcard(model, team)) - .filter(|deployments| !deployments.is_empty()); - if let Some(deployments) = team_owned { + // `_core_wildcard_route_deployments`: the team's own pattern, else the proxy-wide one, + // each tried as spelled and then under the name's provider (`get_pattern`). + let team_owned = team_id.and_then(|team| { + let lookup = crate::lookup::Lookup::new(registry); + [Some(model), caller.wildcard_request] + .into_iter() + .flatten() + .map(|request| (request, lookup.team_wildcard(request, team))) + .find(|(_, deployments)| !deployments.is_empty()) + }); + if let Some((request, deployments)) = team_owned { let routed = deployments.iter().flat_map(|deployment| { let public = deployment.model_info.team_public_model_name.as_deref(); - let pattern = public.unwrap_or(model); - routed_wildcard(pattern, model, std::slice::from_ref(deployment)) + let pattern = public.unwrap_or(request); + routed_wildcard(pattern, request, std::slice::from_ref(deployment)) }); let kept = reserved(model, routed.collect(), caller, now)?; return resolved(ResolveKind::Wildcard, model, kept); @@ -428,6 +440,7 @@ pub fn resolve_request( mod tests { use std::collections::BTreeMap; + use rstest::rstest; use serde_json::{Value, json}; use super::*; @@ -544,8 +557,11 @@ mod tests { #[test] fn a_name_no_pattern_matches_is_tried_under_its_provider() { let registry = registry(wildcard_group()); - assert_eq!(wildcard_name(®istry, "gpt-4o", false), Some("gpt-4o")); - assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false), None); + assert_eq!( + wildcard_name(®istry, "gpt-4o", false, None), + Some("gpt-4o") + ); + assert_eq!(wildcard_name(®istry, "openai/gpt-4o", false, None), None); let caller = Caller { wildcard_request: Some("openai/gpt-4o"), ..Caller::default() @@ -564,6 +580,48 @@ mod tests { assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o"); } + #[rstest] + fn a_bare_name_is_tried_under_its_provider_against_the_teams_own_wildcard() { + let registry = registry(json!([ + {"model_name": "model_name_tw_1", "litellm_params": {"model": "openai/*"}, + "model_info": {"id": "tw", "team_id": "TW", "team_public_model_name": "openai/*"}}, + {"model_name": "g", "litellm_params": {"model": "openai/g"}, "model_info": {"id": "g"}}, + ])); + // No proxy-wide wildcard: only the team's patterns call for the provider spelling. + assert_eq!( + wildcard_name(®istry, "gpt-4o-mini", false, Some("TW")), + Some("gpt-4o-mini") + ); + assert_eq!(wildcard_name(®istry, "gpt-4o-mini", false, None), None); + assert_eq!( + wildcard_name(®istry, "openai/gpt-4o-mini", false, Some("TW")), + None + ); + let resolve = |team_id| { + let caller = Caller { + team_id, + wildcard_request: Some("openai/gpt-4o-mini"), + ..Caller::default() + }; + resolve_request( + ®istry, + "gpt-4o-mini", + false, + caller, + None, + None, + Timestamp::now(), + ) + .unwrap() + }; + let resolution = resolve(Some("TW")).unwrap(); + assert_eq!(resolution.kind, ResolveKind::Wildcard); + assert_eq!(resolution.group, "gpt-4o-mini"); + assert_eq!(ids(Some(resolution.clone())), ["tw"]); + assert_eq!(resolution.deployments[0].params.model, "openai/gpt-4o-mini"); + assert!(resolve(Some("OTHER")).is_none()); + } + #[test] fn a_deployment_named_by_id_refuses_the_endpoint_like_any_other() { let registry = registry(json!([ diff --git a/litellm-rust/crates/router-core/src/select.rs b/litellm-rust/crates/router-core/src/select.rs index 1972b2cbd90..b6ada42df1f 100644 --- a/litellm-rust/crates/router-core/src/select.rs +++ b/litellm-rust/crates/router-core/src/select.rs @@ -281,8 +281,12 @@ impl Core { // No deployment for the name: route to the first default fallback. let default_target = crate::resolve::default_target(&snapshot.settings.dynamic); // `get_pattern`: a name no pattern matches is tried again under its inferred provider. - let wildcard_name = - crate::resolve::wildcard_name(registry, name, request.specific_deployment); + let wildcard_name = crate::resolve::wildcard_name( + registry, + name, + request.specific_deployment, + request.team_id.as_deref(), + ); let wildcard_request = match wildcard_name { Some(model) => { let scope = StrategyScope::Default; From faf4ce39850048aed7eb671b589b9a0d49bb7793 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:47:24 -0800 Subject: [PATCH 09/34] fix(responses): preserve native stream spend on guardrail rejection (#50286) * fix(responses): preserve native stream spend on guardrail rejection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(responses): type deferred logging dispatch once Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(responses): expect guarded rounds on one failure row Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): format guarded billing regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): preserve existing MCP unit formatting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/responses/mcp/stream_round_billing.py | 8 ++----- litellm/responses/streaming_iterator.py | 24 +++++++++++++++++-- .../integration/mcp/test_mcp_llm_endpoints.py | 12 +++++++--- .../mcp/test_aresponses_api_with_mcp.py | 20 ++++++++++++---- 4 files changed, 49 insertions(+), 15 deletions(-) diff --git a/litellm/responses/mcp/stream_round_billing.py b/litellm/responses/mcp/stream_round_billing.py index 5cb256a8ac0..be331a91475 100644 --- a/litellm/responses/mcp/stream_round_billing.py +++ b/litellm/responses/mcp/stream_round_billing.py @@ -1,5 +1,5 @@ import asyncio -from collections.abc import Callable, Coroutine, Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass from datetime import datetime from typing import Final, TypeAlias @@ -297,12 +297,8 @@ class StreamedRoundBilling: self._close(parked) self._bank((self._dispatch(row, billed, self._logging_obj.attempt_scoped_copy()),), row) - def _dispatch( - self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj - ) -> BilledRoundsDispatch | Coroutine[object, object, None]: + def _dispatch(self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj) -> BilledRoundsDispatch: dispatch: Final = self._log_success(_SuccessLog(logger, row, billed.chat_usage)) - if billed.chat_usage is None: - return dispatch return BilledRoundsDispatch(dispatch, billed.spend) @staticmethod diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 249c6370b86..288e4a7119d 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -53,7 +53,7 @@ from litellm.litellm_core_utils.internal_call_metadata import ( requests_background, without_budget_reservation, ) -from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend +from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers @@ -638,6 +638,24 @@ class BaseResponsesAPIStreamingIterator: self._handle_failure(e) raise + def _deferred_logging_dispatch( + self, logging_coroutine: Coroutine[object, object, None], logging_response: object + ) -> BilledRoundsDispatch | Coroutine[object, object, None]: + response: Final[object] = getattr(logging_response, "response", None) + if not isinstance(response, ResponsesAPIResponse): + return logging_coroutine + price: Final = ( + UncachedPrice(0.0, None) + if self._completed_response_cache_hit + else self.logging_obj.uncached_price(response) + ) + spend: Final = BilledRoundsSpend( + ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response.usage), + price.cost or 0.0, + price.cost_breakdown, + ) + return BilledRoundsDispatch(logging_coroutine, spend) + def _log_completed_response(self, *, is_async: bool) -> None: if self._completed_response_logged: return @@ -662,7 +680,9 @@ class BaseResponsesAPIStreamingIterator: if deferred_dispatch_armed: # End-of-stream guardrail scans write guardrail_information after # the terminal event; dispatching now would snapshot metadata early. - self.logging_obj._deferred_stream_complete_args = (logging_coroutine,) + self.logging_obj._deferred_stream_complete_args = ( + self._deferred_logging_dispatch(logging_coroutine, logging_response), + ) else: spawn_logging_task( asyncio.get_running_loop(), logging_coroutine, task_name="Responses stream success logging" diff --git a/tests/integration/mcp/test_mcp_llm_endpoints.py b/tests/integration/mcp/test_mcp_llm_endpoints.py index f6743a8751a..0c97ae62d07 100644 --- a/tests/integration/mcp/test_mcp_llm_endpoints.py +++ b/tests/integration/mcp/test_mcp_llm_endpoints.py @@ -42,7 +42,6 @@ from openai.types.chat import ChatCompletionMessageParam from openai.types.responses.tool_param import Mcp from pydantic import JsonValue, TypeAdapter - pytestmark: Final = [pytest.mark.xdist_group("mcp_llm_private_gateway"), pytest.mark.shares_worker_state] @@ -589,7 +588,11 @@ def _upstream_model(surface: Surface) -> str: }[surface] -PRICES: Final[Mapping[str, JsonValue]] = {"input_cost_per_token": 0.001, "output_cost_per_token": 0.002, "num_retries": 0} +PRICES: Final[Mapping[str, JsonValue]] = { + "input_cost_per_token": 0.001, + "output_cost_per_token": 0.002, + "num_retries": 0, +} @contextmanager @@ -917,7 +920,9 @@ def test_an_open_server_in_another_gateway_cannot_widen_this_gateways_tools(gate register_mcp(scenario, local, "open" + uuid.uuid4().hex[:8], allow_all_keys=True) listed, called = official_client_outcomes(shared, key, "/mcp", f"{foreign_alias}-add", ADD) assert set(listed.tools) == { - f"{foreign_alias}-add", f"{foreign_alias}-multiply", f"{foreign_alias}-fail" + f"{foreign_alias}-add", + f"{foreign_alias}-multiply", + f"{foreign_alias}-fail", }, listed.tools assert called.ok and called.text == str(ADD["a"] + ADD["b"]), called assert [call["body"]["params"]["name"] for call in _peer_add_calls(foreign)] == ["add"] @@ -1528,6 +1533,7 @@ def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_ one_round_rows: Final = _model_rows( one_round.headers["x-litellm-call-id"], _logs_a_failure_billing(_price(FIRST_ROUND) + FIXED_MARGIN) ) + assert _statuses(one_round_rows) == ["failure"], one_round_rows rig.wire.drain() response: Final = rig.send(key, [AUTO], stream=True, guardrails=[STREAM_BLOCK]) assert STREAM_BLOCK_DETAIL in response.text, response.text diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index 037004ab999..a39bc09cee6 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -2722,7 +2722,7 @@ async def test_stream_auto_execute_bills_a_follow_up_cut_after_streamed_text_wit @pytest.mark.asyncio -async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_the_success_row( +async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_one_failure_row( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -2743,10 +2743,22 @@ async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_r if not ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked): ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) await logging_obj.async_failure_handler(blocked, "blocked by output-policy") - await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + await _settle(lambda: len(recorder.failed) > 0) - _assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events)) - assert [row["response_cost"] for row in recorder.failed] == [0.0] + assert recorder.logged == () + assert len(recorder.failed) == 1 + row: Final = recorder.failed[0] + charged_once: Final = _round_cost(_ROUND_ONE_BODY) + _round_cost(_ROUND_TWO_BODY) - _FIXED_MARGIN + assert row["status"] == "failure" + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["response_cost"] == pytest.approx(charged_once) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + client: Final = _final_response(events) + assert "message" in _output_types(client.output) + assert "function_call" not in _output_types(client.output) + assert client.usage is not None + assert (client.usage.input_tokens, client.usage.output_tokens) == (71 + 149, 19 + 5) + assert client.usage.cost == pytest.approx(charged_once) @pytest.mark.asyncio From 588c79618c0e945c6085f8d4e23c645923e873bc Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Thu, 5 Nov 2026 15:50:42 -0800 Subject: [PATCH 10/34] fix(router): give the rust core's priority scheduler the router's redis cache (#50265) The Rust core built its Scheduler without a Redis cache, so the priority queue lived only in each worker's memory and was never shared across workers or instances. Build it on the routing cache's Redis cache, as the Python core did --- litellm/router_rust/invoke/endpoints.py | 7 ++-- ...t_construction_precall_characterization.py | 41 +++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/litellm/router_rust/invoke/endpoints.py b/litellm/router_rust/invoke/endpoints.py index 688b52f44a3..e2c621ccb7d 100644 --- a/litellm/router_rust/invoke/endpoints.py +++ b/litellm/router_rust/invoke/endpoints.py @@ -1907,10 +1907,9 @@ class RustEndpoints: if self._scheduler is None: from litellm.scheduler import Scheduler - self._scheduler = ( - Scheduler(polling_interval=self._polling_interval) # pyright: ignore[reportArgumentType] # as configured - if self._polling_interval is not None - else Scheduler() + self._scheduler = Scheduler( + polling_interval=self._polling_interval, # pyright: ignore[reportArgumentType] # as configured + redis_cache=self._core.routing_cache.redis_cache, ) return self._scheduler diff --git a/tests/unit/test_router/core/test_construction_precall_characterization.py b/tests/unit/test_router/core/test_construction_precall_characterization.py index 2985819cb5b..f58d9a35110 100644 --- a/tests/unit/test_router/core/test_construction_precall_characterization.py +++ b/tests/unit/test_router/core/test_construction_precall_characterization.py @@ -13,6 +13,11 @@ from collections.abc import Callable, Generator, Mapping from typing import Final import pytest +import redis +import redis.asyncio +from fakeredis import FakeRedis, FakeServer +from fakeredis.aioredis import FakeConnection +from redis.asyncio import BlockingConnectionPool import litellm from litellm._logging import verbose_router_logger @@ -23,6 +28,7 @@ from litellm.router_api import ChangeSet, router_init_args from litellm.router_core import RouterCore from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck +from litellm.scheduler import FlowItem from litellm.types.router import ( AlertingConfig, AllowedFailsPolicy, @@ -620,6 +626,41 @@ def test_applying_a_redis_cache_attaches_it_to_the_routing_state_and_the_pins(ma assert core.registry().plugins("prompt_caching_pins").redis_cache is redis_cache +@pytest.fixture +def shared_redis(monkeypatch: pytest.MonkeyPatch) -> FakeServer: + server: Final = FakeServer() + + class Pool(BlockingConnectionPool): + def __init__(self, **kwargs: object) -> None: # kwargs-ok: forwarded to the redis pool + super().__init__(connection_class=FakeConnection, server=server, **kwargs) + + monkeypatch.setattr(redis, "Redis", FakeRedis) + monkeypatch.setattr(redis.asyncio, "BlockingConnectionPool", Pool) + return server + + +async def test_two_workers_on_one_redis_share_the_priority_queue(make_core: MakeCore, shared_redis: FakeServer) -> None: + first: Final = _one(make_core, redis_host="scheduler-shared", redis_port=6399) + second: Final = _one(make_core, redis_host="scheduler-shared", redis_port=6399) + await first.scheduler.add_request(FlowItem(priority=0, request_id="urgent", model_name="g")) # pyright: ignore[reportAttributeAccessIssue] + waiter: Final = FlowItem(priority=5, request_id="waiter", model_name="g") + await second.scheduler.add_request(waiter) # pyright: ignore[reportAttributeAccessIssue] + + assert not await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue] + await first.scheduler.remove_request("urgent", "g") # pyright: ignore[reportAttributeAccessIssue] + assert await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue] + + +async def test_without_redis_each_worker_keeps_its_own_queue(make_core: MakeCore) -> None: + first: Final = _one(make_core) + second: Final = _one(make_core) + await first.scheduler.add_request(FlowItem(priority=0, request_id="urgent", model_name="g")) # pyright: ignore[reportAttributeAccessIssue] + waiter: Final = FlowItem(priority=5, request_id="waiter", model_name="g") + await second.scheduler.add_request(waiter) # pyright: ignore[reportAttributeAccessIssue] + + assert await second.scheduler.poll(waiter, []) # pyright: ignore[reportAttributeAccessIssue] + + def test_cache_responses_installs_a_local_response_cache_on_the_host(make_core: MakeCore, host: ScriptedHost) -> None: core: Final = _one(make_core, cache_responses=True) assert isinstance(host.cache, litellm.Cache) From 05e41909185198e5c85531a514248a70fbd1742f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 5 Nov 2026 15:54:56 -0800 Subject: [PATCH 11/34] refactor(ui): move alerts page text onto i18n (#50271) --- .../src/app/(dashboard)/alerts/page.tsx | 4 +- .../ModelDeprecationAlerts/AlertsInbox.tsx | 33 ++-- .../DeprecationAlertItem.tsx | 10 +- .../DeprecationEmailSetup.tsx | 38 +++-- .../DeprecationModel.tsx | 8 +- .../MigrationDetail.tsx | 114 ++++++++------ .../ModelDeprecationsPanel.tsx | 16 +- .../ModelUsageChart.tsx | 21 +-- .../RecommendedReplacement.tsx | 28 ++-- .../ModelDeprecationAlerts/UsageTags.tsx | 14 +- .../ModelDeprecationAlerts/migrationState.ts | 7 + ui/litellm-dashboard/src/i18n/locales/en.json | 145 ++++++++++++++++++ 12 files changed, 324 insertions(+), 114 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx index ec88a69670e..fe4692192d8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx @@ -1,14 +1,16 @@ "use client"; +import { useTranslation } from "react-i18next"; import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel"; export default function AlertsPage() { + const { t } = useTranslation(); return ( - Model deprecations + {t("alerts.page.tabs.modelDeprecations")} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx index f52af46ca30..58550a6a741 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx @@ -1,19 +1,27 @@ "use client"; import { useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; import type { ColumnDef } from "@tanstack/react-table"; import { DataTable, DataTableToolbar } from "@/components/shared/DataTable"; import { PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; +import i18n from "@/i18n/i18n"; import type { DeprecationAlert } from "./deprecationAlerts"; -import { AlertStatus, effectiveAlertStatus } from "./migrationState"; +import { ALERT_STATUS_LABEL_KEYS, AlertStatus, effectiveAlertStatus } from "./migrationState"; import RecommendedReplacement from "./RecommendedReplacement"; import { DeprecationModel, DeprecationTiming } from "./DeprecationModel"; import { migrationHref } from "./migrationRoute"; +const EMPTY_MESSAGE_KEYS = { + [AlertStatus.Open]: "alerts.inbox.empty.open", + [AlertStatus.Resolved]: "alerts.inbox.empty.resolved", + [AlertStatus.Dismissed]: "alerts.inbox.empty.dismissed", +} as const; + const columns: ColumnDef[] = [ { id: "model", - header: "Model", + header: () => i18n.t("alerts.inbox.columns.model"), cell: ({ row }) => (
@@ -23,7 +31,7 @@ const columns: ColumnDef[] = [ }, { id: "deprecation", - header: "Deprecation", + header: () => i18n.t("alerts.inbox.columns.deprecation"), cell: ({ row }) => (
@@ -31,14 +39,18 @@ const columns: ColumnDef[] = [
), }, - { accessorKey: "requests", header: "Requests · 30d", cell: ({ row }) => row.original.requests.toLocaleString() }, - { accessorKey: "lastUsedDate", header: "Last seen (UTC)" }, + { + accessorKey: "requests", + header: () => i18n.t("alerts.inbox.columns.requests"), + cell: ({ row }) => row.original.requests.toLocaleString(), + }, + { accessorKey: "lastUsedDate", header: () => i18n.t("alerts.inbox.columns.lastSeen") }, { id: "review", header: "", cell: ({ row }) => ( - Review impact → + {i18n.t("alerts.inbox.reviewImpact")} ), }, @@ -59,6 +71,7 @@ export default function AlertsInbox({ onRefresh: () => void; refreshing: boolean; }) { + const { t } = useTranslation(); const [search, setSearch] = useState(""); const counts = { open: 0, resolved: 0, dismissed: 0 }; for (const alert of alerts) counts[effectiveAlertStatus(alert)]++; @@ -75,7 +88,7 @@ export default function AlertsInbox({ {Object.values(AlertStatus).map((value) => ( - {value} + {t(ALERT_STATUS_LABEL_KEYS[value])} {counts[value]} ))} @@ -88,14 +101,14 @@ export default function AlertsInbox({ fillHeight size="compact" isLoading={pending} - loadingMessage="Loading model deprecations…" - noDataMessage={`No ${status} model deprecations in the last 30 days.`} + loadingMessage={t("alerts.inbox.loading")} + noDataMessage={t(EMPTY_MESSAGE_KEYS[status])} toolbar={(table) => (
@@ -22,13 +24,13 @@ export default function DeprecationAlertItem({ alert }: { alert: DeprecationAler

- {alert.requests.toLocaleString()} requests in the last 30 days · Last seen {alert.lastUsedDate} UTC + {t("alerts.alertItem.usage", { requests: alert.requests.toLocaleString(), date: alert.lastUsedDate })}

- Review migration + {t("alerts.alertItem.reviewMigration")} ); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.tsx index b23fd5adadb..0d7185a6669 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationEmailSetup.tsx @@ -1,6 +1,8 @@ "use client"; +import type { ParseKeys } from "i18next"; import { Mail } from "lucide-react"; +import { useTranslation } from "react-i18next"; import { Button, buttonVariants } from "@/components/ui/button"; import { Dialog, @@ -14,44 +16,40 @@ import { import { uiHref } from "@/utils/uiHref"; const emailSettingsHref = `${uiHref("logging-and-alerts")}?tab=email-alerts`; -const setupSteps = [ +const setupSteps: readonly { titleKey: ParseKeys; descriptionKey: ParseKeys }[] = [ { - title: "Connect email delivery", - description: - "Add your SMTP settings and public gateway URL in Email Settings. Send a test email to verify delivery.", + titleKey: "alerts.emailSetup.steps.connect.title", + descriptionKey: "alerts.emailSetup.steps.connect.description", }, { - title: "Enable model deprecation emails", - description: "In Email events, turn on Model Deprecations and save. This is off by default.", + titleKey: "alerts.emailSetup.steps.enable.title", + descriptionKey: "alerts.emailSetup.steps.enable.description", }, { - title: "Owners get two reminders", - description: - "Affected key owners and team admins receive a reminder 30 days before retirement, and one final reminder 1 day before. Recipients are based on actual usage in the last 30 days. Dates are checked in UTC.", + titleKey: "alerts.emailSetup.steps.reminders.title", + descriptionKey: "alerts.emailSetup.steps.reminders.description", }, ]; export default function DeprecationEmailSetup() { + const { t } = useTranslation(); return (
- Give affected owners time to migrate. Email reminders at 30 days and 1 day. + {t("alerts.emailSetup.summary")} - }>Set up email reminders + }>{t("alerts.emailSetup.trigger")}
- Set up model deprecation emails - - Help key owners and team admins act before a provider retires a model. Only their affected usage is - included. - + {t("alerts.emailSetup.title")} + {t("alerts.emailSetup.description")}
    {setupSteps.map((step, index) => ( -
  1. +
  2. -

    {step.title}

    -

    {step.description}

    +

    {t(step.titleKey)}

    +

    {t(step.descriptionKey)}

  3. ))}
- Open Email Settings + {t("alerts.emailSetup.openSettings")}
diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx index dfb0e2b71c3..70f19e5fc85 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx @@ -1,3 +1,4 @@ +import { useTranslation } from "react-i18next"; import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; import type { DeprecationAlert } from "./deprecationAlerts"; import { migrationHref } from "./migrationRoute"; @@ -17,10 +18,11 @@ export function DeprecationModel({ alert }: { alert: DeprecationAlert }) { } export function DeprecationTiming({ alert }: { alert: DeprecationAlert }) { + const { t } = useTranslation(); const days = alert.model.days_until_deprecation; - let label = `Deprecates in ${days} ${days === 1 ? "day" : "days"}`; - if (days === 0) label = "Deprecates today"; - if (days < 0) label = `Deprecated ${Math.abs(days)} ${days === -1 ? "day" : "days"} ago`; + let label = t("alerts.timing.deprecatesIn", { count: days }); + if (days === 0) label = t("alerts.timing.today"); + if (days < 0) label = t("alerts.timing.deprecatedAgo", { count: Math.abs(days) }); return ( ; isViewOnly: boolean; }) { + const { t } = useTranslation(); const [params, setParams] = useQueryStates(migrationUrlParsers); const { data: teams = [] } = useTeams(); const teamNames = useMemo( @@ -144,27 +151,30 @@ export default function MigrationDetail({ () => [ { id: "key", - header: "Virtual key", + header: t("alerts.migrationDetail.columns.key"), cell: ({ row }) => ( - {row.original.metadata.key_alias || "Unnamed key"} + {row.original.metadata.key_alias || t("alerts.migrationDetail.unnamedKey")} ), }, { id: "team", - header: "Team", + header: t("alerts.migrationDetail.columns.team"), cell: ({ row }) => - teamNames.get(row.original.metadata.team_id ?? "") || row.original.metadata.team_id || "No team", + teamNames.get(row.original.metadata.team_id ?? "") || + row.original.metadata.team_id || + t("alerts.migrationDetail.noTeam"), }, { id: "owner", - header: "Owner", - cell: ({ row }) => row.original.metadata.user_email || row.original.metadata.user_id || "Not recorded", + header: t("alerts.migrationDetail.columns.owner"), + cell: ({ row }) => + row.original.metadata.user_email || row.original.metadata.user_id || t("alerts.migrationDetail.notRecorded"), }, { id: "tags", - header: "Usage tags", + header: t("alerts.migrationDetail.columns.tags"), cell: ({ row }) => ( {row.original.metrics.api_requests.toLocaleString()}, }, - { id: "spend", header: "Spend · 30d", cell: ({ row }) => `$${row.original.metrics.spend.toFixed(2)}` }, + { + id: "spend", + header: t("alerts.migrationDetail.columns.spend"), + cell: ({ row }) => `$${row.original.metrics.spend.toFixed(2)}`, + }, ], - [teamNames, tagsByKey, tagQuery.isPending, tagQuery.isError], + [teamNames, tagsByKey, tagQuery.isPending, tagQuery.isError, t], ); const status = effectiveAlertStatus(selected); - const heading = migrationHeading(selected, status); + const heading = migrationHeading(selected, status, t); const copyFollowUp = () => { const teamName = params.team ? teamNames.get(params.team) || params.team : "Team owners"; const count = keyQuery.data?.total_api_keys; @@ -192,23 +206,28 @@ export default function MigrationDetail({ : selected.lastUsedDate; const context = { team: teamName, count, lastSeen, url: window.location.href }; const message = migrationFollowUp(selected, context); - void copyToClipboard(message, "Follow-up copied"); + void copyToClipboard(message, t("alerts.migrationDetail.followUpCopied")); }; return ( <>
- {status} + + {t(ALERT_STATUS_LABEL_KEYS[status])} +

{heading}

- Last seen {selected.lastUsedDate} UTC · {selected.requests.toLocaleString()} requests in the last 30 days + {t("alerts.migrationDetail.lastSeen", { + date: selected.lastUsedDate, + requests: selected.requests.toLocaleString(), + })}

{!isViewOnly && ( <> @@ -219,7 +238,7 @@ export default function MigrationDetail({ onClick={() => statusMutation.mutate(AlertStatus.Resolved)} > - Mark resolved + {t("alerts.migrationDetail.markResolved")} ) : ( )} )} {modelGroupHref(selected.model.model_name) && ( - Model routing + {t("alerts.migrationDetail.modelRouting")} )}
-

- Resolve across all teams once owners confirm migration. New traffic reopens the alert. -

+

{t("alerts.migrationDetail.resolveHint")}

{statusMutation.isError && (

- Could not save alert status. Please retry. + {t("alerts.migrationDetail.saveError")}

)}
-

{params.team ? "Keys in this team" : "Affected keys"}

+

+ {params.team + ? t("alerts.migrationDetail.summary.teamKeys") + : t("alerts.migrationDetail.summary.affectedKeys")} +

{keyQuery.data?.total_api_keys.toLocaleString() ?? "—"}

-

Teams using this model

+

{t("alerts.migrationDetail.summary.teams")}

{teamQuery.data?.length ?? "—"}

-

Deprecation

+

{t("alerts.migrationDetail.summary.deprecation")}

@@ -276,14 +297,12 @@ export default function MigrationDetail({

- Affected keys{" "} + {t("alerts.migrationDetail.keysTable.title")}{" "} {keyQuery.data?.total_api_keys.toLocaleString() ?? "…"}

-

- Keys and owners that used this model in the last 30 days. Tags show actual usage with this model. -

+

{t("alerts.migrationDetail.keysTable.description")}

+
+
+

{t("skills.install.scope")}

+ + + + +
+
+ {publicSkill && ( + + + + + )} +
+

+ {t(publicInstall ? "skills.install.publicHint" : "skills.install.authenticatedHint")} +

+ {command ? ( +
+              {command}
+            
+ ) : ( +

+ {t("skills.install.unavailable")} +

+ )} +

+ {t(publicInstall ? "skills.install.publicRequirements" : "skills.install.authenticatedRequirements")} +

+
+
+

{t("skills.install.shareHint")}

+ +
+ {copyFailed && ( +

+ {t("skills.install.copyFailed")} +

+ )} + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.fixture.py b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.fixture.py new file mode 100644 index 00000000000..d67718b9621 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.fixture.py @@ -0,0 +1,109 @@ +import errno +import json +import os +import pty +import select +import shutil +import signal +import sys +import tempfile +import time +import zipfile +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Final, TypeAlias, cast + +Environment: TypeAlias = Mapping[str, str] + + +def external_command(tool: str) -> None: + root: Final = Path(os.environ["SKILL_FIXTURE_ROOT"]) + args: Final = tuple(sys.argv[1:]) + header: Final = sys.stdin.read() if tool == "curl" and args[args.index("--header") + 1] == "@-" else "" + (root / f"{tool}.json").write_text(json.dumps({"args": args, "env": dict(os.environ), "header": header})) + if tool == "curl": + output: Final = Path(args[args.index("--output") + 1]) + (output.parent / ".skill-install-fixture").write_text(str(root)) + if os.environ["SKILL_FIXTURE_FAIL"] == "1": + sys.exit(22) + with zipfile.ZipFile(output, "w") as archive: + archive.writestr("SKILL.md", "fixture skill") + return + source: Final = Path(args[args.index("add") + 1]) + if "--copy" in args: + shutil.copytree(source, root / "installed") + else: + (root / "installed").symlink_to(source, target_is_directory=True) + + +def run(command: str, fail: bool) -> None: + secret: Final = "private-fixture-key" + with tempfile.TemporaryDirectory() as directory: + root: Final = Path(directory) + (root / "bin").mkdir() + (root / "tmp").mkdir() + for tool in ("curl", "npx"): + executable: Path = root / "bin" / tool + executable.write_text(f"#!{sys.executable}\n" + Path(__file__).read_text()) + executable.chmod(0o700) + environment: Final[Environment] = { + "PATH": f"{root / 'bin'}{os.pathsep}{os.defpath}", + "TMPDIR": str(root / "tmp"), + "SKILL_FIXTURE_ROOT": str(root), + "SKILL_FIXTURE_FAIL": "1" if fail else "0", + "skill_key": "previously-exported-value", + } + child, terminal = pty.fork() + if child == 0: + os.execve("/bin/bash", ["bash", "-c", command], environment) + output: bytes = b"" # rebind-ok: accumulate partial PTY reads + sent: bool = False # rebind-ok: send the key once after its prompt arrives + ended: bool = False # rebind-ok: distinguish PTY EOF from timeout cleanup + deadline: Final = time.monotonic() + 5 + try: + while True: + ready, _, _ = select.select([terminal], [], [], max(0, deadline - time.monotonic())) + if not ready: + raise TimeoutError("Install command did not finish") + try: + chunk: bytes = os.read(terminal, 65536) + except OSError as error: + if error.errno != errno.EIO: + raise + chunk = b"" + if not chunk: + ended = True + break + output += chunk + if not sent and b"LiteLLM API key: " in output: + os.write(terminal, f"{secret}\n".encode()) + sent = True + finally: + if not ended: + os.killpg(child, signal.SIGKILL) + os.close(terminal) + _, status = os.waitpid(child, 0) + curl: Final = cast(Mapping[str, object], json.loads((root / "curl.json").read_text())) # cast-ok: own JSON + record: Final = root / "npx.json" + npx: Final[Mapping[str, object] | None] = json.loads(record.read_text()) if record.exists() else None + curl_args: Final = cast(Sequence[str], curl["args"]) # cast-ok: fixture records argv + installed: Final = root / "installed" / "SKILL.md" + temporary: Final = Path(curl_args[curl_args.index("--output") + 1]).parent + sys.stdout.write(json.dumps({ + "status": os.waitstatus_to_exitcode(status), + "header": curl["header"], + "installArgs": npx["args"] if npx else None, + "telemetry": cast(Environment, npx["env"]).get("DISABLE_TELEMETRY") if npx else None, # cast-ok: own env + "secretLeaked": secret in json.dumps((output.decode(), curl_args, curl["env"], npx)), + "retained": installed.read_text() if installed.exists() else None, + "temporaryRemoved": not temporary.exists(), + })) + marker: Final = temporary / ".skill-install-fixture" + if marker.exists() and marker.read_text() == str(root): + shutil.rmtree(temporary) + + +if Path(sys.argv[0]).name in ("curl", "npx"): + external_command(Path(sys.argv[0]).name) +else: + run(sys.stdin.read(), sys.argv[1] == "fail") diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.test.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.test.ts new file mode 100644 index 00000000000..183457af162 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.test.ts @@ -0,0 +1,107 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { buildSkillInstallCommand, findPublicSkill } from "./skillInstall"; + +const options = { + gatewayUrl: "https://gateway.example.test/proxy", + archiveUrl: "https://gateway.example.test/proxy/skill-hub/skills/review/archive", + name: "review", + agent: "codex" as const, + global: false, + authHeaderName: "X-LiteLLM-Key", +}; + +describe("skill install commands", () => { + it("quotes the public index name without executing shell substitutions", () => { + const name = "review' $(printf injected) `printf injected`"; + const command = buildSkillInstallCommand({ ...options, global: true, publicSkill: { name } }); + const result = spawnSync("bash", ["-c", `npx() { printf '%s\\n' "$DISABLE_TELEMETRY" "$@"; }; ${command}`], { + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "1", + "skills@1.7.6", + "add", + options.gatewayUrl, + "--skill", + name, + "-a", + "codex", + "-g", + ]); + }); + + it.each(["success", "fail"])("executes a private install with safe credentials and cleanup on %s", (outcome) => { + const command = buildSkillInstallCommand({ ...options, agent: "claude-code", global: true })!; + const result = spawnSync( + "python3", + [fileURLToPath(new URL("./skillInstall.fixture.py", import.meta.url)), outcome], + { + input: command, + encoding: "utf8", + timeout: 10_000, + }, + ); + expect(result.status, result.stderr).toBe(0); + const expected = { + status: outcome === "success" ? 0 : 22, + header: "X-LiteLLM-Key: Bearer private-fixture-key\n", + secretLeaked: false, + temporaryRemoved: true, + retained: outcome === "success" ? "fixture skill" : null, + telemetry: outcome === "success" ? "1" : null, + }; + const receipt = JSON.parse(result.stdout) as typeof expected & { installArgs: string[] | null }; + expect(receipt).toMatchObject(expected); + expect(receipt.installArgs).toEqual( + outcome === "success" ? expect.arrayContaining(["--skill", "review", "-a", "claude-code", "-g"]) : null, + ); + }); + + it.each([ + "localhost", + "skills-install.localhost", + "127.0.0.2", + "[::1]", + "gateway.example.test", + "127.attacker.test", + "localhost.attacker.test", + ])("requires HTTPS for authenticated installs outside loopback at %s", (host) => { + const local = ["localhost", "skills-install.localhost", "127.0.0.2", "[::1]"].includes(host); + const httpOptions = { + ...options, + gatewayUrl: `http://${host}`, + archiveUrl: `http://${host}/skill-hub/skills/review/archive`, + }; + expect(buildSkillInstallCommand(httpOptions) === null).toBe(!local); + expect(buildSkillInstallCommand({ ...httpOptions, publicSkill: { name: "review" } })).not.toBeNull(); + }); + + it.each([ + { gatewayUrl: "https://user:password@gateway.example.test/proxy" }, + { archiveUrl: "https://other.example.test/skill.zip" }, + { archiveUrl: "https://gateway.example.test/proxy/skill-hub/submissions/pending/archive" }, + { authHeaderName: "Authorization\r\nX-Other: value" }, + { name: "review\nLITELLM_SKILL\nexit" }, + { gatewayUrl: "file:///private/skill" }, + ])("rejects credentials and values outside the managed archive contract", (invalid) => { + expect(buildSkillInstallCommand({ ...options, ...invalid })).toBeNull(); + }); + + it("uses archive identity instead of a colliding public skill name", () => { + const entry = { name: "review-2", type: "archive" as const, description: "Review", digest: "sha256:abc" }; + const wrong = { ...entry, name: "review", url: "https://other.example.test/skill.zip" }; + const right = { ...entry, url: `${options.archiveUrl}?digest=sha256:abc` }; + expect(findPublicSkill({ $schema: "", skills: [wrong, right] }, options.archiveUrl)).toMatchObject({ + name: right.name, + url: right.url, + }); + expect(findPublicSkill({ $schema: "", skills: [wrong] }, options.archiveUrl)).toBeUndefined(); + expect( + findPublicSkill({ skills: [{ ...right, url: `${right.url}&token=credential` }] }, options.archiveUrl), + ).toBeUndefined(); + expect(findPublicSkill({ skills: null }, options.archiveUrl)).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.ts new file mode 100644 index 00000000000..343bc82961e --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillInstall.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; + +export const SKILL_AGENTS = [ + { value: "codex", label: "Codex" }, + { value: "claude-code", label: "Claude Code" }, + { value: "cursor", label: "Cursor" }, + { value: "github-copilot", label: "GitHub Copilot" }, + { value: "gemini-cli", label: "Gemini CLI" }, +] as const; + +export type SkillAgent = (typeof SKILL_AGENTS)[number]["value"]; +const skillsPackage = "skills@1.7.6"; +const publicIndexSchema = z.object({ + skills: z.array(z.object({ name: z.string(), type: z.literal("archive"), url: z.string() })), +}); + +const shellQuote = (value: string): string => `'${value.replaceAll("'", "'\"'\"'")}'`; + +const parseUrl = (value: string): URL | null => { + try { + const url = new URL(value); + return /^https?:$/.test(url.protocol) && !url.username && !url.password ? url : null; + } catch { + return null; + } +}; + +export const findPublicSkill = (index: unknown, archiveUrl: string) => { + const parsed = publicIndexSchema.safeParse(index); + return parsed.success + ? parsed.data.skills.find((skill) => { + const url = parseUrl(skill.url); + if (!url || url.hash) return false; + return ( + `${url.origin}${url.pathname}` === archiveUrl && [...url.searchParams.keys()].every((key) => key === "digest") + ); + }) + : undefined; +}; + +interface SkillInstallOptions { + gatewayUrl: string; + archiveUrl: string; + name: string; + agent: SkillAgent; + global: boolean; + publicSkill?: { name: string }; + authHeaderName: string; +} + +export function buildSkillInstallCommand(options: SkillInstallOptions): string | null { + const gateway = parseUrl(options.gatewayUrl); + const archive = parseUrl(options.archiveUrl); + if (!gateway || gateway.search || gateway.hash) return null; + if (!archive || archive.origin !== gateway.origin) return null; + if (!/^[a-z0-9-]+$/.test(options.name)) return null; + if (!/^[!#$%&'*+.^_`|~\da-z-]+$/i.test(options.authHeaderName)) return null; + const gatewayUrl = gateway.href.replace(/\/+$/, ""); + if (archive.href !== `${gatewayUrl}/skill-hub/skills/${options.name}/archive`) return null; + const scope = options.global ? " -g" : ""; + if (options.publicSkill) { + return `DISABLE_TELEMETRY=1 npx ${skillsPackage} add ${shellQuote(gatewayUrl)} --skill ${shellQuote(options.publicSkill.name)} -a ${options.agent}${scope}`; + } + const loopback = + /^(?:localhost|\[::1\]|127(?:\.\d{1,3}){3})$/.test(gateway.hostname) || gateway.hostname.endsWith(".localhost"); + if (gateway.protocol !== "https:" && !loopback) return null; + return `bash <<'LITELLM_SKILL' +set -euo pipefail +set +x +skill_tmp=$(mktemp -d) +trap 'rm -rf "$skill_tmp"' EXIT +IFS= read -r -s -p 'LiteLLM API key: ' skill_key { vi.clearAllMocks(); testQueryClient.clear(); + testQueryClient.setQueryDefaults(["public-skill-install"], { refetchOnWindowFocus: true }); skillRequestMock.mockReset().mockRejectedValue(new Error("Unexpected skill request")); fetchMock.mockReset().mockImplementation((input, init) => { const requestUrl = input instanceof Request ? input.url : String(input); @@ -59,6 +62,7 @@ beforeEach(() => { }); afterEach(() => { + focusManager.setFocused(undefined); setGlobalLitellmHeaderName(); vi.restoreAllMocks(); vi.unstubAllGlobals(); @@ -187,3 +191,74 @@ it.each([skill.source, { source: "archive", url: `${getProxyBaseUrl()}/skill-hub expect(await screen.findByText(/No recorded releases yet/)).toBeInTheDocument(); }, ); + +it("copies the selected private install without sharing a session key and reports clipboard results", async () => { + const user = userEvent.setup(); + const pendingCopy = Promise.withResolvers(); + const writeText = vi.spyOn(navigator.clipboard, "writeText").mockReturnValueOnce(pendingCopy.promise); + setGlobalLitellmHeaderName("X-LiteLLM-Key"); + skillRequestMock.mockResolvedValue(Response.json({ detail: "Not Found" }, { status: 404 })); + renderWithProviders( + , + ); + expect(skillRequestMock).not.toHaveBeenCalled(); + fireEvent.click(screen.getByRole("button", { name: "Install / Share" })); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Coding agent" }), "Claude Code"); + fireEvent.click(screen.getByRole("radio", { name: "All projects" })); + const command = screen.getByLabelText("Install command"); + expect(command).toHaveTextContent("-a claude-code --copy --yes -g"); + expect(command).toHaveTextContent("X-LiteLLM-Key"); + expect(command).not.toHaveTextContent("dashboard-token"); + expect(screen.queryByRole("radio", { name: "Public command" })).not.toBeInTheDocument(); + expect(skillRequestMock).toHaveBeenCalledWith( + `${getProxyBaseUrl()}/.well-known/agent-skills/index.json`, + expect.objectContaining({ + credentials: "omit", + redirect: "error", + headers: { Accept: "application/json", "Content-Type": "application/json" }, + }), + ); + fireEvent.click(screen.getByRole("button", { name: "Copy install command" })); + expect(writeText).toHaveBeenCalledWith(command.textContent); + expect(screen.queryByRole("button", { name: "Copied" })).not.toBeInTheDocument(); + pendingCopy.resolve(); + const copiedButton = await screen.findByRole("button", { name: "Copied" }); + writeText.mockRejectedValueOnce(new Error("Clipboard denied")); + fireEvent.click(copiedButton); + expect(await screen.findByRole("alert")).toHaveTextContent("Could not copy the command"); +}); + +it("offers the indexed public skill name and discards its availability when reopened", async () => { + const portablePath = "/skill-hub/skills/my-skill/archive"; + skillRequestMock.mockResolvedValueOnce( + Response.json({ + skills: [{ name: "my-skill-2", type: "archive", url: `${getProxyBaseUrl()}${portablePath}?digest=sha256:abc` }], + }), + ); + renderWithProviders( + , + ); + fireEvent.click(screen.getByRole("button", { name: "Install / Share" })); + fireEvent.click(await screen.findByRole("radio", { name: "Public command" })); + focusManager.setFocused(false); + await act(async () => focusManager.setFocused(true)); + expect(skillRequestMock).toHaveBeenCalledTimes(1); + expect(screen.getByLabelText("Install command")).toHaveTextContent( + `DISABLE_TELEMETRY=1 npx skills@1.7.6 add '${getProxyBaseUrl()}' --skill 'my-skill-2' -a codex`, + ); + fireEvent.click(screen.getByRole("button", { name: "Close" })); + const refreshedIndex = Promise.withResolvers(); + skillRequestMock.mockReturnValueOnce(refreshedIndex.promise); + fireEvent.click(screen.getByRole("button", { name: "Install / Share" })); + expect(screen.queryByRole("radio", { name: "Public command" })).not.toBeInTheDocument(); + refreshedIndex.resolve(Response.json({ skills: [] })); + expect(screen.getByLabelText("Install command")).toHaveTextContent("LiteLLM API key"); +}); diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx index efdf4757c11..c519816f65f 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skill_detail.tsx @@ -19,6 +19,7 @@ import { } from "./helpers"; import type { Plugin, SkillHubSkill } from "./types"; import SkillVersionHistory from "./SkillVersionHistory"; +import SkillInstallDialog from "./SkillInstallDialog"; type SkillDetailData = Pick< SkillHubSkill, @@ -128,6 +129,7 @@ const SkillDetail: React.FC = ({ skill, onBack, accessToken, o const { t } = useTranslation(); const [activeTab, setActiveTab] = useState("overview"); const [copiedKey, setCopiedKey] = useState(null); + const [installOpen, setInstallOpen] = useState(false); const marketplaceRequiresKey = useClaudeCodeMarketplaceRequiresKey(); const visibleTab = activeTab === "downloads" && !isAdmin ? "overview" : activeTab; const portableArchivePath = skill.downloadAvailable @@ -141,6 +143,7 @@ const SkillDetail: React.FC = ({ skill, onBack, accessToken, o getProxyBaseUrl(), typeof window !== "undefined" ? window.location.origin : undefined, ); + const installDialogPath = skill.portable && installOpen ? archivePath : null; const copyToClipboard = (text: string, key: string) => { navigator.clipboard.writeText(text); @@ -199,19 +202,37 @@ const SkillDetail: React.FC = ({ skill, onBack, accessToken, o {/* Header */}
-
+

{skill.name}

- {onEditClick && ( - - )} +
+ {onEditClick && ( + + )} + {skill.portable && archivePath && ( + + )} +
{skill.description && (

{skill.description}

)}
+ {installDialogPath && ( + + )} + {/* Tab bar */}
diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index c22930a8ec5..ec67f4dc09c 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -1670,6 +1670,28 @@ } }, "skills": { + "install": { + "action": "Install / Share", + "title": "Install {{name}}", + "description": "Choose your coding agent and copy the command to your terminal", + "agent": "Coding agent", + "scope": "Install location", + "project": "Current project", + "global": "All projects", + "method": "Install method", + "authenticated": "Authenticated", + "public": "Public command", + "authenticatedHint": "Enter your own LiteLLM key at the terminal prompt. The gateway checks your access before downloading the skill", + "publicHint": "This skill is listed in the gateway's public index and can be installed without a key", + "command": "Install command", + "authenticatedRequirements": "Requires macOS or Linux with Bash, curl, unzip, Node.js 22.20 or later, and npm. Run from your project directory for a project install", + "publicRequirements": "Requires macOS or Linux with Node.js 22.20 or later and npm. Run from your project directory for a project install", + "shareHint": "Share this command with teammates. It contains no key", + "copy": "Copy install command", + "copied": "Copied", + "copyFailed": "Could not copy the command. Select and copy it manually", + "unavailable": "An install command is unavailable for this gateway URL. Authenticated installs require HTTPS or a localhost gateway" + }, "panel": { "title": "Skills", "intro": "Upload or import portable skills. Review proposed changes before publishing them to the Skills Hub.", From 80b1b9b1889845eb686dd64691a14656c21b8a66 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:50:53 -0800 Subject: [PATCH 20/34] fix(router): let type checkers see the real Router constructor arguments (#50278) --- .../management_endpoints/router_settings_endpoints.py | 2 +- litellm/router.py | 11 ++++++++--- litellm/router_utils/handle_error.py | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/management_endpoints/router_settings_endpoints.py b/litellm/proxy/management_endpoints/router_settings_endpoints.py index 9951891fd07..013eb1ebff4 100644 --- a/litellm/proxy/management_endpoints/router_settings_endpoints.py +++ b/litellm/proxy/management_endpoints/router_settings_endpoints.py @@ -68,7 +68,7 @@ def _get_routing_strategies_from_router_class() -> list[str]: Dynamically extract routing strategies from the Router class __init__ method. """ # Get the __init__ signature - sig: Final = inspect.signature(Router.__init__) + sig: Final = inspect.signature(Router) # Get the routing_strategy parameter routing_strategy_param: Final = sig.parameters.get("routing_strategy") diff --git a/litellm/router.py b/litellm/router.py index a5d72daea1e..4b51c16aa81 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -52,7 +52,7 @@ if TYPE_CHECKING: # ``Router(...)``'s parameters, their defaults and its docstring: ``router_arguments`` binds a call # against them and ``Router.__init__`` reports them. The body is never run. def _router_init( - self, + self: object, model_list: list[DeploymentTypedDict] | list[dict[str, Any]] | None = None, ## ASSISTANTS API ## assistants_config: AssistantsTypedDict | None = None, @@ -222,17 +222,22 @@ def _class_descriptor(cls: type, name: str) -> bool: if TYPE_CHECKING: - from typing import overload + from typing import ParamSpec, overload from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject from litellm.types.utils import EmbeddingResponse, LiteLLMBatch, ModelResponse + _P: Final = ParamSpec("_P") + + def _takes_parameters_of(source: Callable[_P, None]) -> Callable[[Callable[..., None]], Callable[_P, None]]: ... + class Router(RustCore, DeprecatedRouterMembers): """What type checkers see: the constructor's parameters and the endpoints' return types. The endpoints are served by the core at run time (litellm/router_rust/invoke/endpoints.py).""" - __init__ = _router_init # pyright: ignore[reportAssignmentType, reportIncompatibleMethodOverride] # Router(...)'s parameters + @_takes_parameters_of(_router_init) + def __init__(self) -> None: ... # pyright: ignore[reportMissingSuperCall] # a type-checking stub @classmethod def from_config(cls, config: RouterConfig, host: PythonHost = python_host) -> "Router": ... diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 6033024123f..97bf4710fca 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -26,7 +26,7 @@ async def send_llm_exception_alert( ctx: "RoutingContext | None", request_kwargs: dict, error_traceback_str: str, - original_exception, + original_exception: Exception, ): """ Only runs if router.slack_alerting_logger is set @@ -34,7 +34,7 @@ async def send_llm_exception_alert( Parameters: ctx (RoutingContext): The router's routing context. - original_exception (Any): The original exception that occurred. + original_exception (Exception): The original exception that occurred. Returns: None From e3272da8485e512192a73ce4767abee46bd32d95 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 5 Nov 2026 16:51:27 -0800 Subject: [PATCH 21/34] fix(ui): keep chat text sent back to the model in English (#50304) The /chat request errors and deep research notice, and Aria's step limit and no answer replies, are written into assistant messages that are sent to the model as history on the next turn. Pin them to English with i18n.getFixedT("en") so a future UI language never changes the model payload. English output is unchanged. --- .../components/chat/runChatRequest.test.ts | 66 ++++++++++++++++++- .../src/components/chat/runChatRequest.ts | 6 +- .../src/components/liteadmin/agent.test.ts | 37 +++++++++++ .../src/components/liteadmin/agent.ts | 4 +- 4 files changed, 107 insertions(+), 6 deletions(-) diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts index 60ab8772112..aca9364c52b 100644 --- a/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts +++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.test.ts @@ -1,5 +1,6 @@ import OpenAI from "openai"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import i18n from "@/i18n/i18n"; import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api"; import { generateChatImages } from "./imageHelpers"; import { toResponsesMessages } from "./attachmentHelpers"; @@ -192,4 +193,67 @@ describe("runChatRequest", () => { expect(options.onUpdate).toHaveBeenCalledExactlyOnceWith({ content: " [stopped]" }); expect(options.onResponseId).toHaveBeenCalledWith(null); }); + + describe("when the UI language is not English", () => { + beforeEach(async () => { + i18n.addResourceBundle( + "de", + "translation", + { + chat: { + runChatRequest: { genericError: "Fehler", fileInputHint: " Hinweis", researchNeedsBackground: "Hinweis" }, + }, + }, + true, + true, + ); + await i18n.changeLanguage("de"); + }); + + afterEach(async () => { + await i18n.changeLanguage("en"); + i18n.removeResourceBundle("de", "translation"); + }); + + it("should keep the generic error in English because it is sent back to the model", async () => { + const options = requestOptions(); + vi.mocked(makeOpenAIResponsesRequest).mockRejectedValue("offline"); + await runChatRequest(options, i18n.t.bind(i18n)); + expect(options.onUpdate).toHaveBeenLastCalledWith({ content: "Something went wrong. Please try again." }); + }); + + it("should keep the file input hint in English because it is sent back to the model", async () => { + const options = requestOptions({ + messages: [ + { + role: "user", + content: "Read the report", + attachments: [ + { + id: "report", + kind: "file", + name: "report.pdf", + mimeType: "application/pdf", + url: "data:application/pdf;base64,JVBERg==", + }, + ], + }, + ], + }); + const error = new OpenAI.APIError(400, { message: "Provider detail" }, "Provider detail", new Headers()); + vi.mocked(makeOpenAIResponsesRequest).mockRejectedValue(error); + await runChatRequest(options, i18n.t.bind(i18n)); + expect(options.onUpdate).toHaveBeenLastCalledWith({ + content: error.message + " Try a model that supports file inputs, or paste the relevant text.", + }); + }); + + it("should keep the deep research notice in English because it is sent back to the model", async () => { + const options = requestOptions({ research: true }); + await runChatRequest(options, i18n.t.bind(i18n)); + expect(options.onUpdate).toHaveBeenCalledExactlyOnceWith({ + content: "Enable background chats before starting deep research.", + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts index 7758f33254d..dc30f7d68f6 100644 --- a/ui/litellm-dashboard/src/components/chat/runChatRequest.ts +++ b/ui/litellm-dashboard/src/components/chat/runChatRequest.ts @@ -31,8 +31,8 @@ function requestErrorMessage( error: unknown, messages: ChatRequestOptions["messages"], mode: ChatRequestOptions["mode"], - t: TFunction = i18n.t.bind(i18n), ): string { + const t = i18n.getFixedT("en"); const message = error instanceof Error ? error.message : t("chat.runChatRequest.genericError"); const hasFile = mode === "chat" && messages.some((item) => item.attachments?.some((attachment) => attachment.kind === "file")); @@ -67,7 +67,7 @@ export async function runChatRequest( try { signal.throwIfAborted(); if (research && !background) { - onUpdate({ content: t("chat.runChatRequest.researchNeedsBackground") }); + onUpdate({ content: i18n.getFixedT("en")("chat.runChatRequest.researchNeedsBackground") }); return; } if (mode === "image") { @@ -125,7 +125,7 @@ export async function runChatRequest( } catch (error: unknown) { onResponseId(null); const stopped = signal.aborted || (error instanceof Error && error.name === "AbortError"); - const errorMessage = requestErrorMessage(error, messages, mode, t); + const errorMessage = requestErrorMessage(error, messages, mode); onUpdate({ content: stopped ? content + " [stopped]" : content || errorMessage }); } finally { if (mcpEvents.length) onUpdate({ mcpEvents }); diff --git a/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts b/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts index e9fae756427..80c21e81e64 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/agent.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ChatCompletion, ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions"; import { createGatewayClient } from "@/components/llm_calls/gateway_client"; import { registerAuthHeaderNameGetter } from "@/lib/http/runtime"; +import i18n from "@/i18n/i18n"; import { getPreferredLiteAdminModel, MAX_INPUT_LENGTH, @@ -889,6 +890,42 @@ describe("SDK conversation and action lifecycle", () => { ); }); +describe("LiteAdmin replies sent back to the model when the UI language is not English", () => { + beforeEach(async () => { + i18n.addResourceBundle( + "de", + "translation", + { aria: { agent: { stepLimit: "Schrittlimit", noAnswer: "Keine Antwort" } } }, + true, + true, + ); + await i18n.changeLanguage("de"); + }); + + afterEach(async () => { + await i18n.changeLanguage("en"); + i18n.removeResourceBundle("de", "translation"); + }); + + it("keeps the step limit reply in English", async () => { + const input = options(); + const model = transport(Array.from({ length: 6 }, () => completion([call("teams_list", teamsArgs)]))); + await runLiteAdmin(input, model.client); + expect(input.onMessage).toHaveBeenCalledExactlyOnceWith( + "I reached the step limit. Any completed actions remain applied; check the relevant page before continuing.", + ); + }); + + it("keeps the no answer reply in English", async () => { + const input = options(); + const model = transport([completion([], null)]); + await runLiteAdmin(input, model.client); + expect(input.onMessage).toHaveBeenCalledExactlyOnceWith( + "The model returned no answer. Try another request or model.", + ); + }); +}); + describe("inference destination", () => { const destinations = [ ["", "/gateway", "https://dashboard.example/ui/", "https://dashboard.example/gateway", false], diff --git a/ui/litellm-dashboard/src/components/liteadmin/agent.ts b/ui/litellm-dashboard/src/components/liteadmin/agent.ts index b9b8420a6e5..9acce836323 100644 --- a/ui/litellm-dashboard/src/components/liteadmin/agent.ts +++ b/ui/litellm-dashboard/src/components/liteadmin/agent.ts @@ -141,8 +141,8 @@ export async function runLiteAdmin(options: LiteAdminOptions, client?: OpenAI): active(); const message = completion.choices[0]?.message; if (message?.tool_calls?.length) { - options.onMessage(i18n.t("aria.agent.stepLimit")); + options.onMessage(i18n.getFixedT("en")("aria.agent.stepLimit")); } else if (!message?.content && !message?.refusal) { - options.onMessage(i18n.t("aria.agent.noAnswer")); + options.onMessage(i18n.getFixedT("en")("aria.agent.noAnswer")); } } From 40a4ebaf6eebf16314eeb478a1fb860cce0b2745 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:52:38 -0800 Subject: [PATCH 22/34] build(deps): bump @azure/msal-node to 7.x in vscode-extension to clear osv-scan (#50198) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- osv-scanner.toml | 5 -- vscode-extension/package-lock.json | 79 ++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 25 deletions(-) diff --git a/osv-scanner.toml b/osv-scanner.toml index 45fb7635088..11c2d4fbd87 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -83,11 +83,6 @@ id = "GHSA-45c3-73f7-pv4m" ignoreUntil = 2026-11-27 reason = "dnspython 2.9.0 (published 2026-10-09) is the only fix and requires Python >= 3.11, so the Python 3.10 resolution stays on 2.8.0 while 3.11+ locks 2.9.0; the advisory is CPU use in dns.name.from_wire_parser and litellm only reaches dnspython through pydantic EmailStr, which never does DNS lookups; remove this entry once 3.10 is dropped or a 3.10-compatible fix ships" -[[IgnoredVulns]] -id = "GHSA-xcx6-jx8p-x3h5" -ignoreUntil = 2026-11-24 -reason = "@azure/msal-node is fixed only in 7.1.0 (published 2026-10-13), but it is dev-only in vscode-extension via @vscode/vsce > @azure/identity, and the latest @azure/identity (4.14.0) still requires msal-node ^6.0.0, so the lock cannot move without forcing a major override; the advisory is in the auth_code flow, which extension packaging does not run; bump and remove this entry once @azure/identity accepts msal-node >= 7.1.0" - [[PackageOverrides]] name = "pillow" version = "12.3.0" diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 80b868e528b..bea38102884 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -128,9 +128,9 @@ } }, "node_modules/@azure/identity": { - "version": "4.13.3", - "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.3.tgz", - "integrity": "sha512-zGQPtqvXPgSA8yfV2CkIQ1qirqk0p9AIVpC5uEkdXQYcKl07QHvyaGYRnZOk0AsQUmxNb4wfkcwY5di8Z5xa9A==", + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.14.1.tgz", + "integrity": "sha512-HHuuFrmN1uuF9D5A6aAo5eHzNFNX4Q+P4SksZqDKse5YaJFFy/zDLG3wXHEV2FVGgetZ3QM8jGURNBtnQYf1NQ==", "dev": true, "license": "MIT", "dependencies": { @@ -143,8 +143,8 @@ "@azure/core-util": "^1.11.0", "@azure/logger": "^1.0.0", "@azure/msal-browser": "^5.5.0", - "@azure/msal-node": "^6.0.0", - "open": "^10.1.0", + "@azure/msal-node": "^7.1.0", + "open": "^11.0.0", "tslib": "^2.2.0" }, "engines": { @@ -189,19 +189,29 @@ } }, "node_modules/@azure/msal-node": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-6.0.1.tgz", - "integrity": "sha512-ixSO1Y/kCVRthRs+hSx/5qkwaunX1/RAePhlMN0wIpIQ4WEZ6AREGGnGd1AP0qspHVsAwzWQKGubpjh50JyJIQ==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-7.2.0.tgz", + "integrity": "sha512-Y87VJsMWF4jiqLqq8XYLb1o+rGc/HacwS3Zydi3bDtaVm5Sa8FlB7YrKrBFKRIYj27RJXAaDkErImMAqtM10Pg==", "dev": true, "license": "MIT", "dependencies": { - "@azure/msal-common": "16.14.1", + "@azure/msal-common": "16.16.0", "jsonwebtoken": "^9.0.0" }, "engines": { "node": ">=20" } }, + "node_modules/@azure/msal-node/node_modules/@azure/msal-common": { + "version": "16.16.0", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.16.0.tgz", + "integrity": "sha512-EZgd8h5nczsdRAQAgXA0A0pC5L/zj43n1jePXvUMgdwXMbH2zhb7b7Hp+Oe7ZOODK/sch2x+soD6FOrtgUbx/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -2219,6 +2229,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-in-ssh": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-in-ssh/-/is-in-ssh-1.0.0.tgz", + "integrity": "sha512-jYa6Q9rH90kR1vKB6NM7qqd1mge3Fx4Dhw5TVlK1MUBqhEOuCagrEHMevNuCcbECmXZ0ThXkRm+Ymr51HwEPAw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-inside-container": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", @@ -2784,19 +2807,21 @@ } }, "node_modules/open": { - "version": "10.2.0", - "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", - "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/open/-/open-11.0.4.tgz", + "integrity": "sha512-++Zlftm0kVLPmzC06t6epuWmcRMDbI4z5P3NNX979WA/k23+NtSOynEGzsVfZwguKw2mi5umVgnBlJQMwRz4Pg==", "dev": true, "license": "MIT", "dependencies": { - "default-browser": "^5.2.1", + "default-browser": "^5.5.1", "define-lazy-prop": "^3.0.0", + "is-in-ssh": "^1.0.0", "is-inside-container": "^1.0.0", - "wsl-utils": "^0.1.0" + "powershell-utils": "^0.2.1", + "wsl-utils": "^1.0.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2915,6 +2940,19 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/powershell-utils": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/powershell-utils/-/powershell-utils-0.2.1.tgz", + "integrity": "sha512-C+y9x90UElAddDZmV4qOx9W53B61PO7cIqWz2dQsWlwswuq4mr8NEwytdGKboYbQlGZ3awrkTeNvcZiZNHnQ8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/proper-lockfile": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", @@ -3505,16 +3543,17 @@ } }, "node_modules/wsl-utils": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", - "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-1.0.1.tgz", + "integrity": "sha512-c16PKLIRxBYpR3jSWsdS7SFp0COm5SOQdiFjY818dYFHJbMGuLBszpF07LBWmOS1uzrHa9VGdynsEgoeKy0oOQ==", "dev": true, "license": "MIT", "dependencies": { - "is-wsl": "^3.1.0" + "is-wsl": "^3.1.0", + "powershell-utils": "^0.2.0" }, "engines": { - "node": ">=18" + "node": ">=20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" From a7f9f892be62591baf46b47eb1610353d0604b54 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 5 Nov 2026 16:57:33 -0800 Subject: [PATCH 23/34] feat(skills): add sortable Skills Hub catalog and publication audit (#50195) * feat(skills): add sortable Skills Hub catalog and publication audit * refactor(skills): remove redundant publisher table column * style(skills): apply dashboard formatting * fix(skills): preserve approval audit and complete hub sorting --- litellm/models/plugin_upload.py | 13 ++ litellm/proxy/_lazy_openapi_snapshot.json | 169 +++++++++++++++++- .../claude_code_marketplace.py | 18 ++ .../skill_hub_endpoints.py | 85 +++++++-- .../public_endpoints/public_endpoints.py | 16 +- litellm/repositories/plugin_publication.py | 72 ++++++-- .../repositories/plugin_upload_repository.py | 2 + .../skill_submission_repository.py | 26 ++- litellm/types/proxy/claude_code_endpoints.py | 5 + litellm/types/proxy/skill_hub.py | 21 ++- .../database/test_skill_submissions.py | 11 +- .../test_claude_code_marketplace.py | 11 +- .../test_skill_hub_endpoints.py | 119 ++++++++++-- .../public_endpoints/test_public_endpoints.py | 11 +- .../repositories/test_plugin_publication.py | 41 ++++- .../test_skill_submission_repository.py | 31 ++++ ...laudeCodePluginsPanel.integration.test.tsx | 50 ++++-- .../ClaudeCodePluginsPanel.test.tsx | 11 +- .../_components/ClaudeCodePluginsPanel.tsx | 34 ++-- .../skills/_components/PluginTable.test.tsx | 29 +-- .../skills/_components/PluginTable.tsx | 37 ++-- .../skills/_components/PluginTableColumns.tsx | 30 +--- .../add_plugin_form.integration.test.tsx | 2 +- .../SkillHubDashboard.integration.test.tsx | 31 ++++ .../components/AIHub/SkillHubDashboard.tsx | 63 ++----- .../components/AIHub/SkillHubTableColumns.tsx | 3 + .../components/CommandPalette/utils.test.ts | 2 +- .../claude_code_plugins/SkillFilters.tsx | 89 +++++++++ .../SkillMetadataColumns.tsx | 95 ++++++++++ .../claude_code_plugins/skillCatalog.ts | 99 ++++++++++ .../components/claude_code_plugins/types.ts | 4 +- .../src/components/leftnav.test.tsx | 2 +- .../PublicAiHub.integration.test.tsx | 47 ++++- .../publicModelHub/ResourcePanes.tsx | 79 +++----- .../publicModelHub/SkillInspector.tsx | 20 ++- ui/litellm-dashboard/src/i18n/locales/en.json | 86 +++++---- ui/litellm-dashboard/src/lib/http/schema.d.ts | 37 +++- 37 files changed, 1229 insertions(+), 272 deletions(-) create mode 100644 tests/unit/repositories/test_skill_submission_repository.py create mode 100644 ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx create mode 100644 ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx create mode 100644 ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts diff --git a/litellm/models/plugin_upload.py b/litellm/models/plugin_upload.py index bd5e85c79cc..95db82dec79 100644 --- a/litellm/models/plugin_upload.py +++ b/litellm/models/plugin_upload.py @@ -41,6 +41,14 @@ class PluginArchiveMetadata: description: str | None = None has_agent_skill_metadata: bool = False updated_at: datetime | None = None + created_at: datetime | None = None + created_by: str | None = None + + +@dataclass(frozen=True, slots=True) +class PluginReview: + approved_by: str + security_scan_passed: bool @dataclass(frozen=True, slots=True) @@ -51,6 +59,7 @@ class PluginPublication: manifest: Mapping[str, object] created_by: str | None content: bytes | None = None + review: PluginReview | None = None @dataclass(frozen=True, slots=True) @@ -66,6 +75,10 @@ class PublishedPlugin: version: str enabled: bool updated_at: datetime + created_at: datetime | None = None + created_by: str | None = None + review: PluginReview | None = None + published_by: str | None = None @dataclass(frozen=True, slots=True) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 7d9e230f8ef..5cb63c98d56 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -7683,6 +7683,17 @@ "PluginListItem": { "description": "Plugin item in list responses.", "properties": { + "approved_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Approved By" + }, "author": { "anyOf": [ { @@ -7715,6 +7726,17 @@ ], "title": "Created At" }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, "description": { "anyOf": [ { @@ -7737,6 +7759,17 @@ ], "title": "Domain" }, + "download_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Download Count" + }, "enabled": { "title": "Enabled", "type": "boolean" @@ -7801,6 +7834,22 @@ ], "title": "Namespace" }, + "published_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published By" + }, + "security_scan_passed": { + "default": false, + "title": "Security Scan Passed", + "type": "boolean" + }, "source": { "additionalProperties": { "type": "string" @@ -59415,6 +59464,14 @@ }, "title": "Skills", "type": "array" + }, + "tags": { + "default": [], + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" } }, "required": [ @@ -59690,6 +59747,17 @@ }, "SkillHubSkill": { "properties": { + "approved_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Approved By" + }, "content_url": { "anyOf": [ { @@ -59701,6 +59769,29 @@ ], "title": "Content Url" }, + "created_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created At" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Created By" + }, "description": { "anyOf": [ { @@ -59732,7 +59823,7 @@ "type": "null" } ], - "description": "Gateway ZIP downloads, included in proxy-admin catalog listings", + "description": "Gateway ZIP downloads, including repeat downloads", "title": "Download Count" }, "download_url": { @@ -59779,6 +59870,22 @@ ], "title": "Namespace" }, + "published_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Published By" + }, + "security_scan_passed": { + "default": false, + "title": "Security Scan Passed", + "type": "boolean" + }, "source": { "additionalProperties": { "type": "string" @@ -60434,6 +60541,66 @@ "title": "Limit", "type": "integer" } + }, + { + "in": "query", + "name": "tag", + "required": false, + "schema": { + "default": "", + "title": "Tag", + "type": "string" + } + }, + { + "in": "query", + "name": "security_scan_passed", + "required": false, + "schema": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "title": "Security Scan Passed" + } + }, + { + "in": "query", + "name": "sort_by", + "required": false, + "schema": { + "default": "name", + "enum": [ + "name", + "download_count", + "updated_at", + "created_at", + "created_by", + "published_by", + "approved_by", + "security_scan_passed" + ], + "title": "Sort By", + "type": "string" + } + }, + { + "in": "query", + "name": "sort_order", + "required": false, + "schema": { + "default": "asc", + "enum": [ + "asc", + "desc" + ], + "title": "Sort Order", + "type": "string" + } } ], "responses": { diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py index 26cb0a2f3e6..3534dd167de 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py @@ -60,6 +60,7 @@ from litellm.proxy.common_utils.claude_code_marketplace_auth import ( ) from litellm.proxy.common_utils.resource_ownership import is_proxy_admin from litellm.proxy.deps.repositories import PluginUploadRepositoryDep +from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.types.proxy.auth.special_headers import SpecialHeaders from litellm.types.proxy.claude_code_endpoints import ( @@ -666,6 +667,7 @@ async def register_plugin( response_model=ListPluginsResponse, ) async def list_plugins( + uploads: PluginUploadRepositoryDep, enabled_only: bool = False, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ) -> ListPluginsResponse: @@ -678,6 +680,17 @@ async def list_plugins( Returns: List of plugins with their metadata. """ + catalog: Final = await list_visible_plugins(enabled_only, user_api_key_dict) + if uploads is None: + raise _error_response(500, CommonProxyErrors.db_not_connected_error.value) + counts: Final = await uploads.download_counts(tuple(plugin.id for plugin in catalog.plugins)) + return ListPluginsResponse( + plugins=[plugin.model_copy(update={"download_count": counts.get(plugin.id, 0)}) for plugin in catalog.plugins], + count=catalog.count, + ) + + +async def list_visible_plugins(enabled_only: bool, user_api_key_dict: UserAPIKeyAuth) -> ListPluginsResponse: try: prisma_client: Final = await get_prisma_client() @@ -693,9 +706,14 @@ async def list_plugins( # Parse manifest to get additional fields manifest = json.loads(p.manifest_json) if p.manifest_json else {} + review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest)) plugin_list.append( PluginListItem( id=p.id, + created_by=p.created_by, + approved_by=review.approved_by if review else None, + published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)), + security_scan_passed=review.security_scan_passed if review else False, name=p.name, version=p.version, description=p.description, diff --git a/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py b/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py index 93d1ace435d..ffddbd31048 100644 --- a/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py +++ b/litellm/proxy/discovery_endpoints/skill_hub_endpoints.py @@ -3,7 +3,8 @@ import hashlib from collections.abc import Mapping from dataclasses import dataclass, replace from datetime import datetime -from typing import Annotated, Final +from itertools import chain +from typing import Annotated, Final, Literal from urllib.parse import quote import httpx @@ -33,7 +34,7 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketp enable_plugin, get_plugin_downloads, header_key_caller, - list_plugins, + list_visible_plugins, plugin_archive_source_matches, plugin_archive_url, plugin_downloader, @@ -65,6 +66,7 @@ from litellm.proxy.discovery_endpoints.skill_import import ( parse_skill_source, select_imported_skill, ) +from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.types.proxy.claude_code_endpoints import PluginListItem from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import AgentSkillsIndexEntry from litellm.types.proxy.skill_hub import ( @@ -78,6 +80,7 @@ from litellm.types.proxy.skill_hub import ( SkillHubSkill, SkillHubVersion, SkillHubVersions, + SkillSortField, ) router: Final = APIRouter(prefix="/skill-hub/skills", tags=["Skills Hub"]) @@ -121,6 +124,11 @@ def _skill_response(request: Request, plugin: PluginListItem, download_count: in namespace=plugin.namespace, keywords=None if plugin.keywords is None else tuple(plugin.keywords), updated_at=None if plugin.updated_at is None else datetime.fromisoformat(plugin.updated_at), + created_at=None if plugin.created_at is None else datetime.fromisoformat(plugin.created_at), + created_by=plugin.created_by, + approved_by=plugin.approved_by, + published_by=plugin.published_by, + security_scan_passed=plugin.security_scan_passed, visibility="public" if plugin.enabled else "private", source={"source": "archive", "url": _download_url(request, plugin.name)} if local_archive else plugin.source, download_url=_download_url(request, plugin.name) if local_archive else None, @@ -137,28 +145,74 @@ async def list_skills( query: str = "", offset: Annotated[int, Query(ge=0)] = 0, limit: Annotated[int, Query(ge=1, le=100)] = 50, + tag: str = "", + security_scan_passed: bool | None = None, + sort_by: SkillSortField = "name", + sort_order: Literal["asc", "desc"] = "asc", ) -> SkillHubList: """Browse the caller's public and granted private skills.""" - catalog: Final = await list_plugins(enabled_only=False, user_api_key_dict=caller) - needle: Final = query.casefold() + catalog: Final = await list_visible_plugins(enabled_only=False, user_api_key_dict=caller) + needle: Final = query.strip().casefold() + selected_tag: Final = tag.strip().casefold() matched: Final = tuple( plugin for plugin in sorted(catalog.plugins, key=lambda item: item.name) - if needle in " ".join((plugin.name, plugin.description or "", *(plugin.keywords or ()))).casefold() + if needle + in " ".join( + ( + plugin.name, + plugin.description or "", + plugin.domain or "", + plugin.namespace or "", + plugin.created_by or "", + plugin.approved_by or "", + plugin.published_by or "", + *(plugin.keywords or ()), + ) + ).casefold() + and (not selected_tag or selected_tag in {word.casefold() for word in plugin.keywords or ()}) + and (security_scan_passed is None or plugin.security_scan_passed == security_scan_passed) ) - page: Final = matched[offset : offset + limit] counts: Final[Mapping[str, int]] = ( - await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page)) - if is_proxy_admin(caller) + await _require_store(uploads).download_counts(tuple(plugin.id for plugin in matched)) + if sort_by == "download_count" else {} ) + ordered: Final = sorted( + matched, key=lambda plugin: _skill_sort_key(plugin, sort_by, counts), reverse=sort_order == "desc" + ) + page: Final = ordered[offset : offset + limit] + page_counts: Final = ( + await _require_store(uploads).download_counts(tuple(plugin.id for plugin in page)) + if sort_by != "download_count" + else counts + ) return SkillHubList( - skills=tuple(_skill_response(request, plugin, counts.get(plugin.id)) for plugin in page), + skills=tuple(_skill_response(request, plugin, page_counts.get(plugin.id)) for plugin in page), count=len(matched), next_offset=offset + limit if offset + limit < len(matched) else None, + tags=tuple( + sorted(set(chain.from_iterable(plugin.keywords or () for plugin in catalog.plugins)), key=str.casefold) + ), ) +def _skill_sort_key(plugin: PluginListItem, field: SkillSortField, counts: Mapping[str, int]) -> str | float: + if field == "download_count": + return counts.get(plugin.id, 0) + if field == "security_scan_passed": + return int(plugin.security_scan_passed) + if field in {"created_at", "updated_at"}: + value: Final = plugin.created_at if field == "created_at" else plugin.updated_at + return datetime.fromisoformat(value).timestamp() if value else 0 + return { + "name": plugin.name, + "created_by": plugin.created_by or "", + "published_by": plugin.published_by or "", + "approved_by": plugin.approved_by or "", + }[field].casefold() + + @router.get("/{name}", response_model=SkillHubSkill) async def get_skill( request: Request, @@ -176,6 +230,7 @@ async def get_skill( def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) -> SkillHubSkill: manifest: Final = MANIFEST_FIELDS.validate_json(metadata.manifest_json or "{}") + review: Final = read_plugin_review(manifest) return _skill_response( request, PluginListItem.model_validate( @@ -187,7 +242,11 @@ def skill_metadata_response(request: Request, metadata: PluginArchiveMetadata) - "source": dict(metadata.source), "has_agent_skill_metadata": metadata.has_agent_skill_metadata, "enabled": metadata.enabled, - "created_at": None, + "created_at": metadata.created_at.isoformat() if metadata.created_at is not None else None, + "created_by": metadata.created_by, + "approved_by": review.approved_by if review else None, + "published_by": read_plugin_publisher(manifest), + "security_scan_passed": review.security_scan_passed if review else False, "updated_at": metadata.updated_at.isoformat() if metadata.updated_at is not None else None, "domain": manifest.get("domain"), "namespace": manifest.get("namespace"), @@ -416,7 +475,11 @@ def skill_publication_response( "version": published.version, "enabled": published.enabled, "has_agent_skill_metadata": has_agent_skill_metadata(publication.manifest), - "created_at": None, + "created_at": published.created_at.isoformat() if published.created_at is not None else None, + "created_by": published.created_by, + "approved_by": published.review.approved_by if published.review else None, + "published_by": published.published_by, + "security_scan_passed": published.review.security_scan_passed if published.review else False, "updated_at": published.updated_at.isoformat(), "domain": publication.manifest.get("domain"), "namespace": publication.manifest.get("namespace"), diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index b8cdea0dea4..cb1e51d0deb 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -24,7 +24,9 @@ from litellm.proxy._types import ( CommonProxyErrors, ) from litellm.proxy.db.health_check_latest import latest_by_model_name +from litellm.proxy.deps.repositories import PluginUploadRepositoryDep from litellm.proxy.utils import get_custom_url +from litellm.repositories.plugin_publication import read_plugin_publisher, read_plugin_review from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets from litellm.types.agents import AgentCard @@ -71,6 +73,9 @@ class _EndpointEntry(TypedDict): class _PluginRow(Protocol): + @property + def created_by(self) -> str | None: ... + @property def id(self) -> str: ... @@ -366,7 +371,7 @@ async def get_mcp_servers(): "/public/skill_hub", tags=["public", "Claude Code Marketplace"], ) -async def public_skill_hub(): +async def public_skill_hub(uploads: PluginUploadRepositoryDep): """Return enabled (public) Claude Code skills, or none when the marketplace requires a key.""" from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import ( MANIFEST_FIELDS, @@ -385,13 +390,22 @@ async def public_skill_hub(): raise HTTPException(status_code=500, detail=CommonProxyErrors.db_not_connected_error.value) try: plugins: Final = await _plugin_table(prisma_client).find_many(where={"enabled": True}) + counts: Final[Mapping[str, int]] = ( + await uploads.download_counts(tuple(plugin.id for plugin in plugins)) if uploads else {} + ) items: Final = [] for plugin in plugins: raw = plugin.manifest_json or {} manifest = json.loads(raw) if isinstance(raw, str) else raw + review = read_plugin_review(MANIFEST_FIELDS.validate_python(manifest)) items.append( PluginListItem( id=plugin.id, + download_count=counts.get(plugin.id), + created_by=plugin.created_by, + approved_by=review.approved_by if review else None, + published_by=read_plugin_publisher(MANIFEST_FIELDS.validate_python(manifest)), + security_scan_passed=review.security_scan_passed if review else False, name=plugin.name, enabled=plugin.enabled, created_at=str(plugin.created_at) if plugin.created_at else None, diff --git a/litellm/repositories/plugin_publication.py b/litellm/repositories/plugin_publication.py index f5fab458c6b..6f94076c3e0 100644 --- a/litellm/repositories/plugin_publication.py +++ b/litellm/repositories/plugin_publication.py @@ -1,15 +1,18 @@ import hashlib import json +from collections.abc import Mapping +from dataclasses import asdict from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from semver import Version from litellm.models.plugin_upload import ( PluginPublication, PluginPublicationBase, PluginPublicationError, + PluginReview, PublishedPlugin, ) from litellm.repositories.prisma_protocols import TableActions @@ -23,6 +26,22 @@ if TYPE_CHECKING: _MANIFEST: Final = TypeAdapter(dict[str, object]) _SOURCE: Final = TypeAdapter(dict[str, str]) +_REVIEW: Final = TypeAdapter(PluginReview) +_PUBLISHER: Final = TypeAdapter[str | None](str | None) + + +def read_plugin_review(manifest: Mapping[str, object]) -> PluginReview | None: + try: + return _REVIEW.validate_python(manifest.get("_litellm_review")) + except ValidationError: + return None + + +def read_plugin_publisher(manifest: Mapping[str, object]) -> str | None: + try: + return _PUBLISHER.validate_python(manifest.get("_litellm_published_by")) + except ValidationError: + return None class PluginPublicationTransaction(Protocol): @@ -122,7 +141,22 @@ async def publish_plugin( ) if isinstance(version, PluginPublicationError): return version - manifest_json: Final = json.dumps({**publication.manifest, "version": version}) + current_manifest: Final = _MANIFEST.validate_json(current.manifest_json or "{}") if current is not None else {} + same_source: Final = current is not None and current_manifest.get("source") == publication.manifest.get("source") + review: Final = publication.review or (read_plugin_review(current_manifest) if same_source else None) + publisher: Final = publication.review.approved_by if publication.review is not None else publication.created_by + manifest_json: Final = json.dumps( + { + **{ + key: value + for key, value in publication.manifest.items() + if key not in {"_litellm_review", "_litellm_published_by"} + }, + "version": version, + **({"_litellm_published_by": publisher} if publisher is not None else {}), + **({"_litellm_review": asdict(review)} if review is not None else {}), + } + ) if publication.content is not None and verified_archive(publication.content, manifest_json) is None: return PluginPublicationError("conflict", "Uploaded archive does not match its published source digest") timestamp: Final = datetime.now(timezone.utc) @@ -141,8 +175,18 @@ async def publish_plugin( await tx.litellm_claudecodepluginupload.create( data={"plugin_name": publication.name, "content": Base64.encode(publication.content)} ) - await _snapshot(tx, created, publication.content, publication.created_by) - return PublishedPlugin(created.id, created.name, version, created.enabled, created.updated_at or timestamp) + await _snapshot(tx, created, publication.content, publisher) + return PublishedPlugin( + created.id, + created.name, + version, + created.enabled, + created.updated_at or timestamp, + created.created_at, + created.created_by, + review, + publisher, + ) previous_archive: Final = await tx.litellm_claudecodepluginupload.find_unique(where={"plugin_name": current.name}) previous_content: Final = ( verified_archive(previous_archive.content.decode(), current.manifest_json) @@ -152,11 +196,7 @@ async def publish_plugin( history_count: Final = await tx.litellm_claudecodepluginversion.count(where={"plugin_id": current.id}, take=1) if history_count == 0: await _snapshot(tx, current, previous_content, None) - unchanged_content: Final = ( - previous_content - if _MANIFEST.validate_json(current.manifest_json or "{}").get("source") == publication.manifest.get("source") - else None - ) + unchanged_content: Final = previous_content if same_source else None content: Final = publication.content if publication.content is not None else unchanged_content updated: Final = await tx.litellm_claudecodeplugintable.update(where={"id": current.id}, data=data) if updated is None: @@ -171,5 +211,15 @@ async def publish_plugin( "update": {"content": Base64.encode(content)}, }, ) - await _snapshot(tx, updated, content, publication.created_by) - return PublishedPlugin(updated.id, updated.name, version, updated.enabled, updated.updated_at or timestamp) + await _snapshot(tx, updated, content, publisher) + return PublishedPlugin( + updated.id, + updated.name, + version, + updated.enabled, + updated.updated_at or timestamp, + updated.created_at, + updated.created_by, + review, + publisher, + ) diff --git a/litellm/repositories/plugin_upload_repository.py b/litellm/repositories/plugin_upload_repository.py index 99e226e442b..8045abf9af9 100644 --- a/litellm/repositories/plugin_upload_repository.py +++ b/litellm/repositories/plugin_upload_repository.py @@ -287,4 +287,6 @@ def _archive_metadata(plugin: "prisma_models.LiteLLM_ClaudeCodePluginTable") -> description=plugin.description, has_agent_skill_metadata=has_agent_skill_metadata(manifest), updated_at=plugin.updated_at, + created_at=plugin.created_at, + created_by=plugin.created_by, ) diff --git a/litellm/repositories/skill_submission_repository.py b/litellm/repositories/skill_submission_repository.py index 829aa0aadbf..03821da042a 100644 --- a/litellm/repositories/skill_submission_repository.py +++ b/litellm/repositories/skill_submission_repository.py @@ -7,12 +7,13 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prisma's generated delegates are dynamic from urllib.parse import urlsplit -from pydantic import BaseModel, ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from litellm.models.plugin_upload import ( PluginPublication, PluginPublicationBase, PluginPublicationError, + PluginReview, PublishedPlugin, uploaded_plugin_source, ) @@ -312,7 +313,9 @@ class SkillSubmissionRepository: if approve and content is not None and publication_base_url else _SOURCE.validate_python(submission.proposal.manifest.get("source")) ) - publication: Final = await _publish_submission(transaction, row, source, content) if approve else None + publication: Final = ( + await _publish_submission(transaction, row, source, content, reviewer) if approve else None + ) if isinstance(publication, PluginPublicationError): return publication reviewed_at: Final = datetime.now(timezone.utc) @@ -346,6 +349,7 @@ async def _publish_submission( row: "prisma_models.LiteLLM_SkillSubmission", source: Mapping[str, str], content: bytes | None, + reviewer: str, ) -> PublishedPlugin | PluginPublicationError: stored: Final = _StoredProposal.model_validate_json(row.proposal_json) proposal: Final = stored.proposal(row.plugin_name) @@ -363,6 +367,24 @@ async def _publish_submission( manifest={**stored.manifest, "source": dict(source)}, content=content, created_by=row.submitted_by, + review=PluginReview(reviewer, scan_passed_for_archive(row.scan_json, source.get("sha256"))), ), base, ) + + +def scan_passed_for_archive(scan_json: str | None, digest: str | None) -> bool: + if not scan_json or not digest: + return False + try: + scan: Final = SkillScan.model_validate_json(scan_json) + except ValidationError: + return False + return ( + scan.status == "completed" + and scan.archive_sha256 == digest + and scan.report is not None + and scan.report.complete + and scan.report.risk_score == 0 + and not scan.report.findings + ) diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py index 5d8479b8ce1..dfbf1de17af 100644 --- a/litellm/types/proxy/claude_code_endpoints.py +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -154,6 +154,11 @@ class PluginListItem(LiteLLMBaseModel): enabled: bool created_at: str | None updated_at: str | None + created_by: str | None = None + approved_by: str | None = None + published_by: str | None = None + security_scan_passed: bool = False + download_count: int | None = None has_agent_skill_metadata: bool = Field(default=False, exclude=True) diff --git a/litellm/types/proxy/skill_hub.py b/litellm/types/proxy/skill_hub.py index d575defcda7..050c3cb7eb8 100644 --- a/litellm/types/proxy/skill_hub.py +++ b/litellm/types/proxy/skill_hub.py @@ -7,6 +7,17 @@ from pydantic import ConfigDict, Field from litellm.models.skill_submission import SubmissionOperation, SubmissionStatus from litellm.types.llms.base import LiteLLMBaseModel +SkillSortField: TypeAlias = Literal[ + "name", + "download_count", + "updated_at", + "created_at", + "created_by", + "published_by", + "approved_by", + "security_scan_passed", +] + class SkillHubSkill(LiteLLMBaseModel): model_config = ConfigDict(frozen=True) @@ -16,6 +27,11 @@ class SkillHubSkill(LiteLLMBaseModel): description: str | None = None version: str | None = None updated_at: datetime | None = None + created_at: datetime | None = None + created_by: str | None = None + approved_by: str | None = None + published_by: str | None = None + security_scan_passed: bool = False visibility: Literal["public", "private"] source: Mapping[str, str] download_url: str | None = None @@ -23,12 +39,11 @@ class SkillHubSkill(LiteLLMBaseModel): domain: str | None = None namespace: str | None = None keywords: tuple[str, ...] | None = None - download_count: int | None = Field( - None, description="Gateway ZIP downloads, included in proxy-admin catalog listings" - ) + download_count: int | None = Field(None, description="Gateway ZIP downloads, including repeat downloads") class SkillHubList(LiteLLMBaseModel): + tags: tuple[str, ...] = () model_config = ConfigDict(frozen=True) skills: tuple[SkillHubSkill, ...] diff --git a/tests/integration/database/test_skill_submissions.py b/tests/integration/database/test_skill_submissions.py index cc00bf6e359..e55e19fb110 100644 --- a/tests/integration/database/test_skill_submissions.py +++ b/tests/integration/database/test_skill_submissions.py @@ -134,6 +134,8 @@ async def test_approval_commits_version_archive_history_and_review(pending_updat assert current is not None published_manifest: Final = { **case.proposal.manifest, + "_litellm_review": {"approved_by": "integration-reviewer", "security_scan_passed": False}, + "_litellm_published_by": "integration-reviewer", "source": { **TypeAdapter(dict[str, str]).validate_python(case.submission.proposal.manifest["source"]), "url": f"https://skills.example.test/claude-code/plugins/{case.original.name}/archive/" @@ -148,17 +150,22 @@ async def test_approval_commits_version_archive_history_and_review(pending_updat ) metadata: Final = await case.publications.get_metadata(case.original.name) assert metadata is not None + assert metadata.created_by == before.created_by == case.original.created_by + assert metadata.created_at == before.created_at + assert metadata.updated_at is not None and before.updated_at is not None + assert metadata.updated_at > before.updated_at assert await case.publications.get_content(metadata) == case.proposal.content history: Final = await case.publications.versions(metadata, 10, 0) assert tuple((version.version, version.description, version.archive_available) for version in history) == ( ("1.0.1", case.proposal.description, True), ("1.0.0", case.original.description, True), ) + assert history[0].published_by == "integration-reviewer" + assert history[1].published_by == case.original.created_by for version, publication, manifest in zip( - history, (case.proposal, case.original), (published_manifest, case.original.manifest), strict=True + history, (case.proposal, case.original), (published_manifest, {**case.original.manifest, "_litellm_published_by": case.original.created_by}), strict=True ): assert version.manifest == {**manifest, "version": version.version} - assert version.published_by == publication.created_by assert await case.publications.version_content(metadata, version.id) == publication.content diff --git a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py index 7e5b3a7abd7..a1be416f4cb 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py +++ b/tests/unit/proxy/anthropic_endpoints/test_claude_code_marketplace.py @@ -74,6 +74,7 @@ def _make_mock_prisma(): record.description = data.get("description") record.manifest_json = data.get("manifest_json", "{}") record.enabled = data.get("enabled", True) + record.created_by = data.get("created_by") record.created_at = data.get("created_at") record.updated_at = data.get("updated_at") store[data["name"]] = record @@ -221,7 +222,7 @@ async def test_metadata_edits_preserve_uploaded_source_and_reject_a_changed_http await table.create(data={"name": _UPLOAD_NAME, "manifest_json": json.dumps({"source": _UPLOAD_SOURCE})}) request: Final = UpdatePluginRequest(source=_UPLOAD_SOURCE, description="Updated description", version="1.0.0") response: Final = await update_plugin(uploads=_uploads(), plugin_name=_UPLOAD_NAME, request=request, user_api_key_dict=_USER) - expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description", "version": "1.0.0"} + expected: Final = {"name": _UPLOAD_NAME, "source": _UPLOAD_SOURCE, "description": "Updated description", "version": "1.0.0", "_litellm_published_by": _USER.user_id} assert response.plugin.description == "Updated description" assert await _read_stored_manifest(_UPLOAD_NAME) == expected with pytest.raises(HTTPException) as error: @@ -354,7 +355,7 @@ async def _register_public_and_private_plugins() -> None: async def _listed_names(user: UserAPIKeyAuth) -> set[str]: - response = await list_plugins(user_api_key_dict=user) + response = await list_plugins(uploads=_uploads(), user_api_key_dict=user) return {plugin.name for plugin in response.plugins} @@ -474,7 +475,7 @@ async def test_update_plugin_propagates_installation_preference_to_get_and_list( ) plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER) - listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0] + listed_plugin = (await list_plugins(uploads=_uploads(), user_api_key_dict=_USER)).plugins[0] assert response.plugin.installation_preference == "auto_install" assert plugin["installation_preference"] == "auto_install" @@ -514,7 +515,7 @@ async def test_stored_unknown_installation_preference_is_not_served(stored_value ) marketplace = json.loads((await get_marketplace(request=MagicMock())).body) - listed_plugin = (await list_plugins(user_api_key_dict=_USER)).plugins[0] + listed_plugin = (await list_plugins(uploads=_uploads(), user_api_key_dict=_USER)).plugins[0] plugin = await get_plugin(plugin_name=name, user_api_key_dict=_USER) assert marketplace["plugins"] == [{"name": name, "source": _ARCHIVE_SOURCE}] @@ -1288,7 +1289,7 @@ async def test_skill_upload_creates_metadata_and_bytes_atomically_and_rejects_du "url": f"http://test/claude-code/plugins/{_UPLOAD_NAME}/archive/{digest}.zip", "sha256": digest, } - assert json.loads(stored["manifest_json"]) == {**metadata, "source": source} + assert json.loads(stored["manifest_json"]) == {**metadata, "source": source, "_litellm_published_by": stored["created_by"]} assert stored == { "name": _UPLOAD_NAME, "description": "My notes", "version": "2.0.0", "manifest_json": stored["manifest_json"], "created_by": litellm.proxy.proxy_server.litellm_proxy_admin_name, diff --git a/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py b/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py index 8127fc80bab..7912df3939b 100644 --- a/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py +++ b/tests/unit/proxy/discovery_endpoints/test_skill_hub_endpoints.py @@ -196,25 +196,25 @@ def _assert_portable(response: httpx.Response) -> None: assert stat.S_IMODE(archive.getinfo("scripts/check.sh").external_attr >> 16) == 0o755 -@pytest.mark.parametrize("key, expected", ((MASTER_KEY, 7), (PLAIN_KEY, None))) -async def test_catalog_counts_are_admin_only_and_batched_for_the_page( - hub: HubClient, key: str, expected: int | None, +@pytest.mark.parametrize("route", (f"{HUB}?limit=1", "/claude-code/plugins")) +@pytest.mark.parametrize("key", (MASTER_KEY, PLAIN_KEY)) +async def test_catalog_counts_are_batched_for_the_page( + hub: HubClient, key: str, route: str, ) -> None: assert (await _upload(hub, _zip(FILES))).status_code == 200 record: Final = _stored_upload(hub) hub.plugins.find_many.return_value = [record, record.model_copy(update={"id": "other", "name": "zzz-other"})] hub.database.query_raw.reset_mock() hub.database.query_raw.return_value = [{"id": record.id, "download_count": 7}] - response: Final = await hub.client.get(f"{HUB}?limit=1", headers=_headers(key)) + response: Final = await hub.client.get(route, headers=_headers(key)) assert response.status_code == 200, response.text - assert [(item["id"], item["download_count"]) for item in response.json()["skills"]] == [(record.id, expected)] - if expected is None: - hub.database.query_raw.assert_not_awaited() - else: - assert hub.database.query_raw.await_count == 1 - assert hub.database.query_raw.await_args.args[1] == (record.id,) - assert (await hub.client.get(f"{HUB}?offset=2", headers=_headers(key))).json()["skills"] == [] - assert hub.database.query_raw.await_count == 1 + items: Final = response.json()["plugins" if route == "/claude-code/plugins" else "skills"] + expected: Final = {record.id: 7, "other": 0} if route == "/claude-code/plugins" else {record.id: 7} + assert {item["id"]: item["download_count"] for item in items} == expected + assert hub.database.query_raw.await_count == 1 + assert set(hub.database.query_raw.await_args.args[1]) == set(expected) + assert (await hub.client.get(f"{HUB}?offset=2", headers=_headers(key))).json()["skills"] == [] + assert hub.database.query_raw.await_count == 1 @pytest.mark.parametrize("operation", ("upload", "import")) @@ -806,3 +806,98 @@ async def test_configured_scan_persists_result_without_changing_manual_approval( assert (await hub.client.get(path, headers=_headers(MASTER_KEY))).json() == before approved: Final = await hub.client.put(path + "/approve", headers=_headers(MASTER_KEY)) assert approved.status_code == 200 and approved.json()["submission"]["status"] == "approved" + + +async def test_catalog_filters_metadata_before_pagination_and_preserves_tag_facets(hub: HubClient) -> None: + hub.plugins.find_many.return_value = [ + prisma_models.LiteLLM_ClaudeCodePluginTable( + id=name, name=name, enabled=True, anonymous_download_count=0, created_by="author@example.test", + created_at=datetime(2026, 1, 2), updated_at=datetime(2026, 2, 3), + manifest_json=json.dumps({ + "source": {"source": "github", "repo": "example/skills"}, + "keywords": tags, + "_litellm_published_by": "publisher@example.test", + **({"_litellm_review": {"approved_by": "reviewer@example.test", "security_scan_passed": scanned}} if approved else {}), + }), + ) + for name, tags, approved, scanned in ( + ("a-other", ["support"], False, False), + ("b-review", ["engineering"], True, False), + ("c-review", ["engineering"], True, True), + ) + ] + response = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={ + "query": " REVIEWER ", "tag": "ENGINEERING", "limit": 1, "offset": 1, + }) + assert response.status_code == 200 + body = response.json() + assert body["count"] == 2 + assert body["tags"] == ["engineering", "support"] + assert [skill["name"] for skill in body["skills"]] == ["c-review"] + skill = body["skills"][0] + assert skill["created_by"] == "author@example.test" + assert skill["published_by"] == "publisher@example.test" + assert skill["approved_by"] == "reviewer@example.test" + assert skill["created_at"].startswith("2026-01-02") + assert skill["updated_at"].startswith("2026-02-03") + assert skill["security_scan_passed"] is True + scanned = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={"security_scan_passed": "true"}) + assert [skill["name"] for skill in scanned.json()["skills"]] == ["c-review"] + unscanned = await hub.client.get(HUB, headers=_headers(MASTER_KEY), params={"security_scan_passed": "false"}) + assert [skill["name"] for skill in unscanned.json()["skills"]] == ["a-other", "b-review"] + assert unscanned.json()["skills"][0]["approved_by"] is None + + +@pytest.mark.parametrize("has_issues", (False, True)) +async def test_admin_approval_does_not_override_scan_result(pending_hub: HubClient, has_issues: bool) -> None: + hub: Final = pending_hub + pending: Final = await _upload(hub, _zip(FILES), MEMBER_KEY, version=None) + receipt: Final = pending.json() + submission: Final = hub.database.litellm_skillsubmission.rows[0] + digest: Final = hashlib.sha256(submission.archive_content.decode()).hexdigest() + await hub.database.litellm_skillsubmission.update( + where={"id": submission.id}, data={"scan_json": json.dumps({ + "id": "scan", "requested_by": "admin", "started_at": "2026-01-01T00:00:00Z", + "expires_at": "2026-01-01T00:03:00Z", "status": "completed", "archive_sha256": digest, + "report": {"scanner_version": "test", "scan_mode": "static-only", "complete": True, + "risk_score": 0, "findings": [{"rule_id": "rule", "severity": "LOW", + "message": "Issue", "file": "SKILL.md"}] if has_issues else []}, + })}, + ) + approved: Final = await hub.client.put( + f"/skill-hub/submissions/{receipt['submission_id']}/approve", headers=_headers(MASTER_KEY), + ) + assert approved.status_code == 200 + skill: Final = approved.json()["skill"] + assert skill["security_scan_passed"] is not has_issues + assert skill["created_by"] == MEMBER_KEY + assert skill["approved_by"] == approved.json()["submission"]["reviewed_by"] + assert skill["published_by"] == skill["approved_by"] + _stored_upload(hub) + listed: Final = (await hub.client.get(HUB, headers=_headers(MASTER_KEY))).json()["skills"][0] + assert listed["security_scan_passed"] is not has_issues + assert listed["published_by"] == skill["published_by"] + + +@pytest.mark.parametrize("field,order,expected", [ + ("name", "asc", "a"), ("name", "desc", "c"), + ("download_count", "desc", "a"), ("download_count", "asc", "b"), + ("updated_at", "desc", "c"), ("updated_at", "asc", "b"), +]) +async def test_catalog_sorts_all_matches_before_pagination( + hub: HubClient, field: str, order: str, expected: str, +) -> None: + hub.plugins.find_many.return_value = [ + prisma_models.LiteLLM_ClaudeCodePluginTable( + id=name, name=name, enabled=True, anonymous_download_count=0, updated_at=datetime(2026, 1, day), + manifest_json=json.dumps({"source": {"source": "github", "repo": "example/skills"}}), + ) for name, day in (("a", 2), ("b", 1), ("c", 3)) + ] + hub.database.query_raw.return_value = [ + {"id": name, "download_count": count} for name, count in (("a", 8), ("b", 0), ("c", 3)) + ] + response = await hub.client.get(HUB, headers=_headers(PLAIN_KEY), params={ + "sort_by": field, "sort_order": order, "limit": 1, "offset": 0, + }) + assert response.status_code == 200, response.text + assert [skill["name"] for skill in response.json()["skills"]] == [expected] diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index dd3ffad4727..d1afcf80ec9 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1,3 +1,5 @@ +from litellm.proxy.deps.repositories import get_plugin_upload_repository +from litellm.repositories.plugin_upload_repository import PluginUploadRepository import asyncio import json import re @@ -1804,7 +1806,6 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_marketplace import register_plugin - from litellm.repositories.plugin_upload_repository import PluginUploadRepository from litellm.types.proxy.claude_code_endpoints import RegisterPluginRequest def create(data: Mapping[str, object]) -> LiteLLM_ClaudeCodePluginTable: @@ -1826,6 +1827,7 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: prisma.db.litellm_claudecodepluginversion.create = AsyncMock() app: Final = FastAPI() app.include_router(router) + app.dependency_overrides[get_plugin_upload_repository] = lambda: PluginUploadRepository(prisma) with patch("litellm.proxy.proxy_server.prisma_client", prisma): asyncio.run( @@ -1843,8 +1845,10 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin"), ) ) + prisma.db.query_raw.return_value = [{"id": "plugin-id", "download_count": 12}] response: Final = TestClient(app).get("/public/skill_hub") + assert response.json()["plugins"][0]["download_count"] == 12 assert response.status_code == 200 assert [(p["name"], p["installation_preference"]) for p in response.json()["plugins"]] == [ ("security-review", preference) @@ -1854,6 +1858,7 @@ def test_public_skill_hub_echoes_registered_installation_preference(preference: _INTERNAL_PLUGIN: Final = SimpleNamespace( id="plugin-id", name="deploy-helper", + created_by=None, enabled=True, created_at=None, updated_at=None, @@ -1874,8 +1879,11 @@ def _skill_hub_response(find_many: AsyncMock, general_settings: Mapping[str, obj table.find_many = find_many prisma: Final = MagicMock() prisma.db.litellm_claudecodeplugintable = table + prisma.writer_db = prisma.db + prisma.db.query_raw = AsyncMock(return_value=[]) app: Final = FastAPI() app.include_router(router) + app.dependency_overrides[get_plugin_upload_repository] = lambda: PluginUploadRepository(prisma) with ( patch("litellm.proxy.proxy_server.prisma_client", prisma), @@ -1909,6 +1917,7 @@ def test_public_skill_hub_500_body_carries_no_exception_text() -> None: def test_public_skill_hub_without_a_database_answers_db_not_connected() -> None: app: Final = FastAPI() app.include_router(router) + app.dependency_overrides[get_plugin_upload_repository] = lambda: None with ( patch("litellm.proxy.proxy_server.prisma_client", None), diff --git a/tests/unit/repositories/test_plugin_publication.py b/tests/unit/repositories/test_plugin_publication.py index 0a784553d09..7a96b0fd5a7 100644 --- a/tests/unit/repositories/test_plugin_publication.py +++ b/tests/unit/repositories/test_plugin_publication.py @@ -16,6 +16,7 @@ from litellm.models.plugin_upload import ( PluginPublication, PluginPublicationBase, PluginPublicationError, + PluginReview, PublishedPlugin, ) from litellm.repositories.plugin_publication import ( @@ -144,6 +145,44 @@ async def test_first_legacy_update_preserves_its_original_snapshot( } assert replacement.kwargs["data"] == { "plugin_id": current.id, "version": result.version, "description": "replacement", - "manifest_json": json.dumps({"source": source, "version": result.version}), + "manifest_json": json.dumps({"source": source, "version": result.version, "_litellm_published_by": "admin"}), "published_at": result.updated_at, "published_by": "admin", "content": None, "archive_available": False, } + + +@pytest.mark.parametrize("same_source", (True, False)) +@pytest.mark.parametrize("review", [None, PluginReview("reviewer@example.test", False)]) +@pytest.mark.asyncio +async def test_publication_preserves_only_trusted_review_for_unchanged_source( + publication_tx: MagicMock, review: PluginReview | None, same_source: bool, +) -> None: + original = publication_tx.litellm_claudecodeplugintable.find_unique.return_value + stored_review = {"approved_by": "original-reviewer", "security_scan_passed": True} + stored_manifest = {**json.loads(original.manifest_json), "_litellm_review": stored_review} + current = original.model_copy(update={"manifest_json": json.dumps(stored_manifest)}) + publication_tx.litellm_claudecodeplugintable.find_unique.return_value = current + + async def update(where: Mapping[str, str], data: Mapping[str, object]) -> LiteLLM_ClaudeCodePluginTable: + return current.model_copy(update=dict(data)) + + publication_tx.litellm_claudecodeplugintable.update.side_effect = update + result = await publish_plugin( + cast(PluginPublicationTransaction, publication_tx), + PluginPublication("skill", None, "updated", { + "_litellm_review": {"approved_by": "forged", "security_scan_passed": True}, + "source": stored_manifest["source"] if same_source else {"source": "github", "repo": "example/skills"}, + }, "author", review=review), + PluginPublicationBase(current.id, current.updated_at), + ) + assert isinstance(result, PublishedPlugin) + manifest = json.loads(publication_tx.litellm_claudecodeplugintable.update.call_args.kwargs["data"]["manifest_json"]) + assert manifest["_litellm_published_by"] == (review.approved_by if review else "author") + expected = ( + {"approved_by": review.approved_by, "security_scan_passed": review.security_scan_passed} if review + else stored_review if same_source else None + ) + assert manifest.get("_litellm_review") == expected + assert (None if result.review is None else { + "approved_by": result.review.approved_by, "security_scan_passed": result.review.security_scan_passed, + }) == expected + assert json.loads(publication_tx.litellm_claudecodepluginversion.create.call_args.kwargs["data"]["manifest_json"]).get("_litellm_review") == expected diff --git a/tests/unit/repositories/test_skill_submission_repository.py b/tests/unit/repositories/test_skill_submission_repository.py new file mode 100644 index 00000000000..7187cdbe514 --- /dev/null +++ b/tests/unit/repositories/test_skill_submission_repository.py @@ -0,0 +1,31 @@ +import json + +import pytest + +from litellm.repositories.skill_submission_repository import scan_passed_for_archive + + +@pytest.mark.parametrize("status,digest,report,expected", [ + ("completed", "current", {"complete": True, "risk_score": 0, "findings": []}, True), + ("running", "current", {"complete": True, "risk_score": 0, "findings": []}, False), + ("failed", "current", {"complete": True, "risk_score": 0, "findings": []}, False), + ("completed", "previous", {"complete": True, "risk_score": 0, "findings": []}, False), + ("completed", "current", {"complete": False, "risk_score": 0, "findings": []}, False), + ("completed", "current", None, False), + ("completed", "current", {"complete": True, "risk_score": 90, "findings": []}, False), + ("completed", "current", {"complete": True, "risk_score": 0, "findings": [{"rule_id": "rule", "severity": "LOW", "message": "Issue", "file": "SKILL.md"}]}, False), + ("completed", "current", {"complete": True, "risk_score": 0}, False), +]) +def test_scan_pass_requires_clean_complete_report_for_published_archive( + status: str, digest: str, report: dict[str, object] | None, expected: bool, +) -> None: + assert scan_passed_for_archive(json.dumps({ + "id": "scan", "requested_by": "admin", "started_at": "2026-01-01T00:00:00Z", + "expires_at": "2026-01-01T00:03:00Z", "status": status, "archive_sha256": digest, + "report": None if report is None else {"scanner_version": "test", "scan_mode": "static-only", **report}, + }), "current") is expected + + +@pytest.mark.parametrize("record", [None, "", "{}", "null", "not-json"]) +def test_missing_or_invalid_scan_is_not_reported_as_passed(record: str | None) -> None: + assert scan_passed_for_archive(record, "current") is False diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx index 359fb12dd95..9a260bdfc65 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.integration.test.tsx @@ -1,7 +1,8 @@ +import userEvent from "@testing-library/user-event"; import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; -import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { setGlobalLitellmHeaderName } from "@/components/networking"; import type { SkillHubSkill, SkillSubmission } from "@/components/claude_code_plugins/types"; import { toast } from "@/lib/toast"; @@ -61,6 +62,7 @@ const pending: SkillSubmission = { const fetchMock = vi.fn(); const queryDefaults = testQueryClient.getDefaultOptions(); let catalog: SkillHubSkill[]; +let catalogRequests: URL[]; let submissions: SkillSubmission[]; let pageRequests: { limit: number; offset: number }[]; let writes: { path: string; method: string; body: RequestInit["body"] }[]; @@ -161,6 +163,7 @@ beforeEach(() => { testQueryClient.clear(); vi.clearAllMocks(); catalog = [published]; + catalogRequests = []; submissions = [pending]; pageRequests = []; writes = []; @@ -180,14 +183,17 @@ beforeEach(() => { return respondToWrite(path, body); } if (path === "/skill-hub/skills") { + catalogRequests.push(url); const limit = Number(url.searchParams.get("limit")); const offset = Number(url.searchParams.get("offset")); const skills = catalog.slice(offset, offset + limit); - return Response.json({ + const result = { + tags: ["review", "engineering"], skills, count: skills.length, next_offset: offset + limit < catalog.length ? offset + limit : null, - }); + }; + return Response.json(result); } if (path === `/skill-hub/skills/${published.name}`) return Response.json(catalog.find((skill) => skill.name === published.name)); @@ -250,7 +256,7 @@ it.each(["current", "retained"])("refreshes published counts after a %s download catalog = [{ ...published, download_count: 1 }]; fireEvent.click(button); await waitFor(() => expect(button).toBeEnabled()); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); expect(await screen.findByRole("cell", { name: "1" })).toBeInTheDocument(); }); @@ -266,7 +272,7 @@ it("paginates the neutral catalog and refreshes detail for a recreated skill", a renderWithProviders(); fireEvent.click(await screen.findByText(published.name)); expect(await screen.findByRole("button", { name: "Edit Skill" })).toBeInTheDocument(); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); catalog = [ { ...published, id: "filler", name: "catalog-0" }, ...catalog.slice(1), @@ -281,7 +287,7 @@ it("paginates the neutral catalog and refreshes detail for a recreated skill", a fireEvent.click(screen.getByText(published.name)); expect(await screen.findByRole("button", { name: "Edit Skill" })).toBeInTheDocument(); expect(screen.getByText("Recreated instructions")).toBeInTheDocument(); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); fireEvent.click(screen.getByRole("button", { name: "Previous" })); await waitFor(() => expect(screen.getByRole("button", { name: "Previous" })).toBeDisabled()); }); @@ -394,7 +400,7 @@ it("allows a member submission, explains approval and refreshes their pending li await waitFor(() => expect(screen.getByRole("tab", { name: "Published skills" })).toHaveAttribute("aria-selected", "true"), ); - fireEvent.click(screen.getByRole("button", { name: "+ Add Skill" })); + fireEvent.click(screen.getByRole("button", { name: "+ Register your skill" })); fireEvent.click(screen.getByRole("radio", { name: "Import URL" })); fireEvent.change(screen.getByRole("textbox", { name: "Source URL" }), { target: { value: "https://github.com/example/new-skill" }, @@ -593,10 +599,10 @@ it("refreshes detail after a review conflict and removes actions when another re it("keeps an effective Admin viewer read-only while allowing full submission inspection", async () => { renderWithProviders(); - expect(screen.getByRole("button", { name: "+ Add Skill" })).toBeDisabled(); + expect(screen.getByRole("button", { name: "+ Register your skill" })).toBeDisabled(); fireEvent.click(await screen.findByText(published.name)); expect(screen.queryByRole("button", { name: "Edit Skill" })).not.toBeInTheDocument(); - fireEvent.click(screen.getByText("Skills")); + fireEvent.click(screen.getByText("Skills Hub")); fireEvent.click(screen.getByRole("tab", { name: "Submissions" })); fireEvent.click(await screen.findByRole("button", { name: "View details" })); expect(await screen.findByText("Currently published")).toBeInTheDocument(); @@ -643,7 +649,7 @@ it.each(["role", "read-only"])("does not reuse a published admin snapshot after expect(screen.queryByText("Approved instructions")).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Edit Skill" })).not.toBeInTheDocument(); expect(await screen.findByRole("alert")).toHaveTextContent("Could not refresh published skills"); - expect(screen.queryByRole("columnheader", { name: "Downloads" })).not.toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Downloads" })).toBeInTheDocument(); }); it("does not reuse admin review metadata when the same account loses its role and detail refresh fails", async () => { @@ -762,3 +768,27 @@ it.each([200, 409])("refreshes the running scan after POST %s and ignores an old fireEvent.click(screen.getByRole("button", { name: "Approve and publish" })); expect(await screen.findByText("Approved")).toBeInTheDocument(); }); + +it("applies tag and scan filters across the catalog and resets pagination when searching", async () => { + const user = userEvent.setup(); + catalog = [published, { ...published, id: "second", name: "second-skill" }]; + renderWithProviders(, { + searchParams: { skills_page: "2", skills_page_size: "1" }, + }); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("offset")).toBe("1")); + await screen.findByRole("cell", { name: "Approved instructions" }); + await user.click(screen.getByRole("button", { name: "Downloads" })); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("sort_by")).toBe("download_count")); + expect(catalogRequests.at(-1)?.searchParams.get("offset")).toBe("0"); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by tag" }), "engineering"); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("tag")).toBe("engineering")); + expect(catalogRequests.at(-1)?.searchParams.get("offset")).toBe("0"); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Security scan" }), "Security scan: Yes"); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("security_scan_passed")).toBe("true")); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { + target: { value: "reviewer@example.test" }, + }); + await waitFor(() => expect(catalogRequests.at(-1)?.searchParams.get("query")).toBe("reviewer@example.test")); + expect(catalogRequests.at(-1)?.searchParams.get("tag")).toBe("engineering"); + expect(catalogRequests.at(-1)?.searchParams.get("security_scan_passed")).toBe("true"); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx index a0e785268ea..dbdf5ca86d7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.test.tsx @@ -87,7 +87,16 @@ describe("ClaudeCodePluginsPanel loading state", () => { resolveFetch({ skills: [], count: 0, next_offset: null }); expect(await screen.findByText("table-loaded")).toBeInTheDocument(); - expect(mockListSkillHubSkills).toHaveBeenCalledWith("sk-test", { limit: 50, offset: 0 }); + const expectedQuery = { + limit: 50, + offset: 0, + query: "", + tag: "", + security_scan_passed: undefined, + sort_by: "name", + sort_order: "asc", + }; + expect(mockListSkillHubSkills).toHaveBeenCalledWith("sk-test", expectedQuery); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx index 6efa511ce49..6497c0922d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/ClaudeCodePluginsPanel.tsx @@ -1,3 +1,6 @@ +import type { SkillListQuery } from "@/components/claude_code_plugins/types"; +import { SkillFilters } from "@/components/claude_code_plugins/SkillFilters"; +import { useSkillFilters, skillSortQuery } from "@/components/claude_code_plugins/skillCatalog"; import React, { useState } from "react"; import { useQuery, useQueryClient, type QueryKey } from "@tanstack/react-query"; import { Trans, useTranslation } from "react-i18next"; @@ -21,7 +24,6 @@ import { toast } from "@/lib/toast"; import { isPendingSkill, type SkillHubSkill, type SkillMutationResponse } from "@/components/claude_code_plugins/types"; import { useUrlTab } from "@/hooks/useUrlTab"; import SkillSubmissions from "./SkillSubmissions"; -import { useUrlTableState, type UrlTableStateOptions } from "@/components/shared/DataTable/useUrlTableState"; interface ClaudeCodePluginsPanelProps { accessToken: string | null; @@ -29,14 +31,7 @@ interface ClaudeCodePluginsPanelProps { isViewOnly?: boolean; } -const CATALOG_STATE: UrlTableStateOptions = { - keyPrefix: "skills_", - defaultPageSize: 50, - maxPageSize: 100, - sortFields: ["name"], - defaultSort: { id: "name", desc: false }, - filterColumns: [], -}; +const scanResultFilter = (scan: string): boolean | undefined => (scan === "all" ? undefined : scan === "yes"); const SKILLS_TABS = ["skills", "submissions"] as const; @@ -117,11 +112,20 @@ const SkillsPanelContent: React.FC = ({ const { readOnly, canWrite, isAdmin, adminView } = skillPermissions(accessToken, userRole, isViewOnly); const queryScope = ["skill-hub", accessToken, userRole, readOnly] as const; const showForm = Boolean(isAddModalVisible || editingSkill) && canWrite; - const { pagination, onPaginationChange } = useUrlTableState(CATALOG_STATE); + const filters = useSkillFilters(); + const { pagination, onPaginationChange, sorting, onSortingChange } = filters; const { pageIndex, pageSize } = pagination; + const catalogQuery: SkillListQuery & { limit: number; offset: number } = { + limit: pageSize, + offset: pageIndex * pageSize, + query: filters.search, + tag: filters.tag, + security_scan_passed: scanResultFilter(filters.scan), + ...skillSortQuery(sorting), + }; const plugins = useQuery({ - queryKey: [...queryScope, "published", pageIndex, pageSize], - queryFn: () => listSkillHubSkills(accessToken!, { limit: pageSize, offset: pageIndex * pageSize }), + queryKey: [...queryScope, "published", catalogQuery], + queryFn: () => listSkillHubSkills(accessToken!, catalogQuery), enabled: Boolean(accessToken), }); const pluginsList = plugins.data?.skills ?? []; @@ -193,6 +197,9 @@ const SkillsPanelContent: React.FC = ({ +
+ +
{plugins.isError && (
{t("skills.panel.loadFailed")} @@ -204,6 +211,9 @@ const SkillsPanelContent: React.FC = ({ { [12345, "12,345"], ] as const)("renders headers and download count %s", (download_count, display) => { render(); - for (const header of ["Skill Name", "Version", "Description", "Domain", "Public", "Updated at", "Downloads"]) { + for (const header of [ + "Skill Name", + "Tags", + "Created by", + "Approved by", + "Created", + "Last updated", + "Security scan", + "Downloads", + ]) { expect(screen.getByText(header)).toBeInTheDocument(); } - expect(within(screen.getByRole("row", { name: /newer-skill/ })).getByText(display)).toBeInTheDocument(); + expect(within(screen.getByRole("row", { name: /newer-skill/ })).getAllByText(display)[0]).toBeInTheDocument(); }); it("should display the empty state when data is empty", () => { render(); - expect(screen.getByText("No skills found")).toBeInTheDocument(); - }); - - it("should sort by name ascending by default", () => { - render(); - const rows = screen.getAllByRole("row").slice(1); - expect(within(rows[0]).getByText("newer-skill")).toBeInTheDocument(); - expect(within(rows[1]).getByText("older-skill")).toBeInTheDocument(); + expect(screen.getByText("No skills registered yet")).toBeInTheDocument(); }); it("should call onPluginClick with the plugin ID when the skill name is clicked", async () => { @@ -89,7 +93,8 @@ describe("PluginTable", () => { it("should show whether the skill is public", () => { render(); expect(screen.getByText("Yes")).toBeInTheDocument(); - expect(screen.getByText("No")).toBeInTheDocument(); + const privateRow = screen.getByRole("row", { name: /older-skill/ }); + expect(within(privateRow).getAllByText("No")).toHaveLength(2); }); it("should delete a skill through the actions menu when admin", async () => { @@ -111,7 +116,7 @@ describe("PluginTable", () => { it("should hide the delete action for non-admins but keep copy available", async () => { const user = userEvent.setup(); render(); - expect(screen.queryByRole("columnheader", { name: "Downloads" })).not.toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Downloads" })).toBeInTheDocument(); await user.click(screen.getByTestId("plugin-actions-newer-skill")); expect(await screen.findByTestId("plugin-action-copy")).toBeInTheDocument(); expect(screen.queryByTestId("plugin-action-delete")).not.toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx index db61f80c690..943b34c7023 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTable.tsx @@ -1,8 +1,8 @@ "use client"; -import { SortingState } from "@tanstack/react-table"; +import { type OnChangeFn, type SortingState } from "@tanstack/react-table"; import { Inbox } from "lucide-react"; -import React, { useMemo, useState } from "react"; +import React, { useMemo } from "react"; import { useTranslation } from "react-i18next"; import { DataTable } from "@/components/shared/DataTable"; @@ -12,30 +12,43 @@ import { getPluginTableColumns } from "./PluginTableColumns"; interface PluginTableProps { pluginsList: SkillHubSkill[]; + sorting: SortingState; + onSortingChange: OnChangeFn; isLoading: boolean; + filtered?: boolean; onDeleteClick: (pluginName: string, displayName: string) => void; isAdmin: boolean; onPluginClick: (pluginId: string) => void; } -const DEFAULT_SORTING: SortingState = [{ id: "name", desc: false }]; - -function EmptyState() { +function EmptyState({ filtered }: { filtered: boolean }) { const { t } = useTranslation(); return (
-
{t("skills.table.empty.title")}
-
{t("skills.table.empty.description")}
+
+ {t(filtered ? "aiHub.skillHubDashboard.emptyState.noMatchTitle" : "skills.table.empty.title")} +
+
+ {t(filtered ? "aiHub.skillHubDashboard.emptyState.noMatchBody" : "skills.table.empty.description")} +
); } -const PluginTable: React.FC = ({ pluginsList, isLoading, onDeleteClick, isAdmin, onPluginClick }) => { +const PluginTable: React.FC = ({ + pluginsList, + sorting, + onSortingChange, + isLoading, + filtered = false, + onDeleteClick, + isAdmin, + onPluginClick, +}) => { const { t } = useTranslation(); - const [sorting, setSorting] = useState(DEFAULT_SORTING); const columns = useMemo( () => getPluginTableColumns({ isAdmin, onPluginClick, onDeleteClick }), @@ -48,12 +61,12 @@ const PluginTable: React.FC = ({ pluginsList, isLoading, onDel paginationMode="none" columns={columns} getRowId={(plugin, index) => plugin.id || String(index)} - sortingMode="client" + sortingMode="server" sorting={sorting} - onSortingChange={setSorting} + onSortingChange={onSortingChange} isLoading={isLoading} loadingMessage={t("skills.table.loading")} - noDataMessage={} + noDataMessage={} size="compact" /> ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx index f2055b77969..c838ea5deda 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/PluginTableColumns.tsx @@ -1,11 +1,13 @@ "use client"; +import { getSkillMetadataColumns } from "@/components/claude_code_plugins/SkillMetadataColumns"; + import { ColumnDef } from "@tanstack/react-table"; import { Copy, MoreHorizontal, Trash2 } from "lucide-react"; import { useTranslation } from "react-i18next"; import { DataTableSortHeader } from "@/components/shared/DataTable"; -import { DateCell, IdentityCell, StatusBadge } from "@/components/shared/table_cells"; +import { IdentityCell, StatusBadge } from "@/components/shared/table_cells"; import i18n from "@/i18n/i18n"; import { SkillHubSkill } from "@/components/claude_code_plugins/types"; import { buttonVariants } from "@/components/ui/button"; @@ -143,31 +145,7 @@ export const getPluginTableColumns = ({ /> ), }, - { - id: "updated_at", - accessorKey: "updated_at", - sortingFn: "datetime", - meta: { title: i18n.t("skills.table.columns.updatedAt") }, - header: ({ column }) => , - size: 160, - enableSorting: true, - cell: ({ row }) => , - }, - ...(isAdmin - ? ([ - { - id: "download_count", - accessorKey: "download_count", - meta: { title: i18n.t("skills.table.columns.downloads") }, - header: i18n.t("skills.table.columns.downloads"), - size: 120, - enableSorting: false, - cell: ({ row }) => ( - {row.original.download_count?.toLocaleString(i18n.language) ?? "—"} - ), - }, - ] satisfies ColumnDef[]) - : []), + ...getSkillMetadataColumns(), { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx index 778bd8e309a..9547db39ca1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/skills/_components/add_plugin_form.integration.test.tsx @@ -15,7 +15,7 @@ const file = new File(["skill archive"], "personal-skill.zip", { type: "applicat const chooseFile = (selected = file) => fireEvent.change(screen.getByLabelText("Skill ZIP file"), { target: { files: [selected] } }); const change = (label: string, value: string) => fireEvent.change(screen.getByLabelText(label), { target: { value } }); -const submit = () => fireEvent.click(screen.getByRole("button", { name: "Add Skill" })); +const submit = () => fireEvent.click(screen.getByRole("button", { name: "Register skill" })); const openImport = () => { renderWithProviders(); fireEvent.click(screen.getByRole("radio", { name: "Import URL" })); diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx new file mode 100644 index 00000000000..8f97f63cd2f --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.integration.test.tsx @@ -0,0 +1,31 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import { renderWithProviders } from "@/../tests/test-utils"; +import type { Plugin } from "@/components/claude_code_plugins/types"; +import SkillHubDashboard from "./SkillHubDashboard"; + +const skills: Plugin[] = [ + { id: "alpha", name: "alpha", source: {}, category: "Writing", domain: "Support", enabled: false, download_count: 1 }, + { + id: "beta", + name: "beta", + source: {}, + category: "Analysis", + domain: "Engineering", + enabled: true, + download_count: 7, + }, +]; + +describe("Skills Hub sorting", () => { + it.each(["Category", "Domain", "Status", "Downloads"])("sorts by %s through the table header", async (header) => { + const user = userEvent.setup(); + renderWithProviders(); + expect(screen.getAllByRole("row")[1]).toHaveTextContent("alpha"); + await user.click(screen.getByRole("button", { name: header, exact: true })); + await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("beta")); + await user.click(screen.getByRole("button", { name: header, exact: true })); + await waitFor(() => expect(screen.getAllByRole("row")[1]).toHaveTextContent("alpha")); + }); +}); diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx index 00f2da9171d..3099da3476e 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubDashboard.tsx @@ -1,15 +1,21 @@ import React, { useMemo, useState } from "react"; -import { SortingState } from "@tanstack/react-table"; -import { Inbox, Search, X } from "lucide-react"; +import { Inbox } from "lucide-react"; import { useTranslation } from "react-i18next"; import { Plugin } from "@/components/claude_code_plugins/types"; import { DataTable } from "@/components/shared/DataTable"; import { getSkillHubTableColumns } from "@/components/AIHub/SkillHubTableColumns"; import SkillDetail, { legacySkillDetail } from "@/components/claude_code_plugins/skill_detail"; -import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; +import { SkillFilters } from "@/components/claude_code_plugins/SkillFilters"; +import { + useSkillFilters, + matchesSkillFilters, + skillTags, + SKILL_SORT_FIELDS, +} from "@/components/claude_code_plugins/skillCatalog"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; const ALL_DOMAINS = "__all_domains__"; +const DASHBOARD_SORT_FIELDS = [...SKILL_SORT_FIELDS, "category", "domain", "enabled"]; interface SkillHubDashboardProps { skills: Plugin[]; @@ -50,10 +56,9 @@ const SkillHubDashboard: React.FC = ({ onPublishSuccess, }) => { const { t } = useTranslation(); - const [search, setSearch] = useState(""); + const filters = useSkillFilters(DASHBOARD_SORT_FIELDS); const [domainFilter, setDomainFilter] = useState(undefined); const [selectedSkill, setSelectedSkill] = useState(null); - const [sorting, setSorting] = useState([{ id: "name", desc: false }]); // Derived stats const totalSkills = skills.length; @@ -69,19 +74,10 @@ const SkillHubDashboard: React.FC = ({ if (domainFilter) { result = result.filter((s) => (s.domain || "General") === domainFilter); } - if (search.trim()) { - const q = search.toLowerCase(); - result = result.filter( - (s) => - s.name.toLowerCase().includes(q) || - s.description?.toLowerCase().includes(q) || - s.domain?.toLowerCase().includes(q) || - s.namespace?.toLowerCase().includes(q) || - s.keywords?.some((k) => k.toLowerCase().includes(q)), - ); - } - return result; - }, [skills, search, domainFilter]); + return result.filter((skill) => + matchesSkillFilters(skill, { search: filters.search, tag: filters.tag, scan: filters.scan }), + ); + }, [skills, filters.search, filters.tag, filters.scan, domainFilter]); const columns = useMemo(() => getSkillHubTableColumns({ onSkillClick: setSelectedSkill }), []); @@ -93,7 +89,7 @@ const SkillHubDashboard: React.FC = ({ [domains, t], ); - const hasActiveFilter = search.trim().length > 0 || domainFilter != null; + const hasActiveFilter = filters.active || domainFilter != null; if (selectedSkill) { return ( @@ -131,7 +127,7 @@ const SkillHubDashboard: React.FC = ({

{publicPage ? t("aiHub.skillHubDashboard.allPublicSkills") : t("aiHub.skillHubDashboard.allSkills")}

-
+
- - - - - setSearch(e.target.value)} - /> - {search !== "" && ( - - setSearch("")} - > - - - - )} - +
= ({ columns={columns} getRowId={(skill, index) => skill.id || String(index)} sortingMode="client" - sorting={sorting} - onSortingChange={setSorting} + sorting={filters.sorting} + onSortingChange={filters.onSortingChange} isLoading={isLoading} loadingMessage={t("aiHub.skillHubDashboard.loading")} noDataMessage={} diff --git a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx index 766c4243c45..76ba9f6e592 100644 --- a/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/SkillHubTableColumns.tsx @@ -1,5 +1,7 @@ "use client"; +import { getSkillMetadataColumns } from "@/components/claude_code_plugins/SkillMetadataColumns"; + import { ColumnDef } from "@tanstack/react-table"; import { Copy, ExternalLink, Info, MoreHorizontal } from "lucide-react"; import { useTranslation } from "react-i18next"; @@ -159,6 +161,7 @@ export const getSkillHubTableColumns = ({ onSkillClick }: SkillHubTableColumnsDe /> ), }, + ...getSkillMetadataColumns(), { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, diff --git a/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts b/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts index fceee39e943..d1f14137577 100644 --- a/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts +++ b/ui/litellm-dashboard/src/components/CommandPalette/utils.test.ts @@ -59,7 +59,7 @@ describe("visibleMenuGroups", () => { }); expect(flattenNavItems(visible).find(({ route }) => route === "skills")).toMatchObject({ - label: "Skills", + label: "Skills Hub", route: "skills", }); }, diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx new file mode 100644 index 00000000000..49088ec1364 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillFilters.tsx @@ -0,0 +1,89 @@ +import { Search } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import { Button } from "@/components/ui/button"; +import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import type { ScanFilter } from "./skillCatalog"; + +interface SkillFiltersProps { + search: string; + tag: string; + scan: ScanFilter; + tags: string[]; + active: boolean; + setSearch: (value: string) => void; + setTag: (value: string) => void; + setScan: (value: ScanFilter) => void; + clear: () => void; +} + +export function SkillFilters({ + search, + tag, + scan, + tags, + active, + setSearch, + setTag, + setScan, + clear, +}: SkillFiltersProps) { + const { t } = useTranslation(); + const tagItems = [ + { value: "", label: t("skills.filters.allTags") }, + ...[...new Set([...tags, ...(tag ? [tag] : [])])].map((value) => ({ value, label: value })), + ]; + const scanItems = [ + { value: "all", label: t("skills.filters.allScans") }, + { value: "yes", label: t("skills.filters.scanned") }, + { value: "no", label: t("skills.filters.notScanned") }, + ]; + return ( +
+ + + + + setSearch(event.target.value)} + /> + + + + {active && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx new file mode 100644 index 00000000000..649a1696a9e --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/SkillMetadataColumns.tsx @@ -0,0 +1,95 @@ +import { DataTableSortHeader } from "@/components/shared/DataTable/DataTableSortHeader"; +import type { ColumnDef } from "@tanstack/react-table"; +import { DateCell, StatusBadge } from "@/components/shared/table_cells"; +import { Badge } from "@/components/ui/badge"; +import i18n from "@/i18n/i18n"; +import type { SkillCatalogMetadata } from "./skillCatalog"; + +const IDENTITY_LABELS = { + created_by: "skills.table.columns.createdBy", + approved_by: "skills.table.columns.approvedBy", +} as const; + +export function getSkillMetadataColumns(): ColumnDef[] { + return [ + { + id: "download_count", + accessorKey: "download_count", + header: ({ column }) => , + size: 120, + sortUndefined: "last", + accessorFn: (skill) => skill.download_count ?? undefined, + cell: ({ row }) => ( + {row.original.download_count?.toLocaleString(i18n.language) ?? "—"} + ), + }, + { + id: "keywords", + header: i18n.t("skills.table.columns.tags"), + size: 180, + enableSorting: false, + cell: ({ row }) => ( +
+ {row.original.keywords?.length + ? row.original.keywords.map((tag) => ( + + {tag} + + )) + : "—"} +
+ ), + }, + ...(["created_by", "approved_by"] as const).map( + (field): ColumnDef => ({ + id: field, + accessorKey: field, + size: 190, + enableSorting: true, + header: ({ column }) => , + cell: ({ row }) => ( + + {row.original[field] || i18n.t("skills.table.notRecorded")} + + ), + }), + ), + ...(["created_at", "updated_at"] as const).map( + (field): ColumnDef => ({ + id: field, + accessorKey: field, + size: 140, + enableSorting: true, + accessorFn: (skill) => (skill[field] ? new Date(skill[field]!).getTime() : undefined), + sortUndefined: "last", + header: ({ column }) => ( + + ), + cell: ({ row }) => , + }), + ), + { + id: "security_scan_passed", + accessorKey: "security_scan_passed", + size: 130, + enableSorting: true, + header: ({ column }) => ( + {i18n.t("skills.table.columns.securityScan")}} + /> + ), + cell: ({ row }) => ( + + + + ), + }, + ]; +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts new file mode 100644 index 00000000000..ec057a00984 --- /dev/null +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/skillCatalog.ts @@ -0,0 +1,99 @@ +import type { SkillListQuery } from "./types"; +import type { SortingState } from "@tanstack/react-table"; +import { parseAsString, parseAsStringLiteral, useQueryStates } from "nuqs"; +import { useUrlTableState, type UrlTableStateOptions } from "@/components/shared/DataTable/useUrlTableState"; +import { matchesSearchTerm } from "@/utils/searchUtils"; + +const SCAN_FILTERS = ["all", "yes", "no"] as const; +export type ScanFilter = (typeof SCAN_FILTERS)[number]; + +export interface SkillCatalogMetadata { + created_by?: string | null; + approved_by?: string | null; + published_by?: string | null; + created_at?: string | null; + updated_at?: string | null; + security_scan_passed?: boolean; + download_count?: number | null; + keywords?: string[] | null; +} + +export const SKILL_SORT_FIELDS = [ + "name", + "download_count", + "created_at", + "updated_at", + "created_by", + "approved_by", + "security_scan_passed", +] as const satisfies readonly NonNullable[]; + +export function skillSortQuery(sorting: SortingState): Pick { + return { + sort_by: SKILL_SORT_FIELDS.find((field) => field === sorting[0]?.id) ?? "name", + sort_order: sorting[0]?.desc ? "desc" : "asc", + }; +} + +const CATALOG_STATE: UrlTableStateOptions = { + keyPrefix: "skills_", + defaultPageSize: 50, + maxPageSize: 100, + sortFields: SKILL_SORT_FIELDS, + defaultSort: { id: "name", desc: false }, + filterColumns: [], +}; +const FILTER_PARSERS = { + tag: parseAsString.withDefault(""), + scan: parseAsStringLiteral(SCAN_FILTERS).withDefault("all"), +}; +const FILTER_OPTIONS = { urlKeys: { tag: "skills_filter_tag", scan: "skills_filter_scan" } }; + +export function useSkillFilters(sortFields: readonly string[] = SKILL_SORT_FIELDS) { + const table = useUrlTableState({ ...CATALOG_STATE, sortFields }); + const [filters, setFilters] = useQueryStates(FILTER_PARSERS, FILTER_OPTIONS); + const update = (next: { tag?: string; scan?: ScanFilter } | null) => { + void setFilters(next); + table.onPaginationChange({ ...table.pagination, pageIndex: 0 }); + }; + return { + ...table, + ...filters, + setTag: (tag: string) => update({ tag }), + setScan: (scan: ScanFilter) => update({ scan }), + clear: () => { + update(null); + table.setSearch(""); + }, + active: Boolean(table.search.trim() || filters.tag || filters.scan !== "all"), + }; +} + +export function skillTags(skills: SkillCatalogMetadata[]): string[] { + return [...new Set(skills.flatMap((skill) => skill.keywords ?? []))].sort((a, b) => a.localeCompare(b)); +} + +export function matchesSkillFilters( + skill: SkillCatalogMetadata & { + name: string; + description?: string | null; + domain?: string | null; + namespace?: string | null; + }, + filters: { search: string; tag: string; scan: ScanFilter }, +): boolean { + const matchesQuery = matchesSearchTerm(filters.search, [ + skill.name, + skill.description, + skill.domain, + skill.namespace, + skill.created_by, + skill.approved_by, + skill.published_by, + ...(skill.keywords ?? []), + ]); + const matchesTag = + !filters.tag || (skill.keywords ?? []).some((tag) => tag.toLowerCase() === filters.tag.toLowerCase()); + const matchesScan = filters.scan === "all" || Boolean(skill.security_scan_passed) === (filters.scan === "yes"); + return matchesQuery && matchesTag && matchesScan; +} diff --git a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts index c0560c09951..7574500e693 100644 --- a/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts +++ b/ui/litellm-dashboard/src/components/claude_code_plugins/types.ts @@ -3,7 +3,7 @@ * API request/response shapes are synced from the generated OpenAPI types in @/lib/http/schema. */ -import type { components } from "@/lib/http/schema"; +import type { components, operations } from "@/lib/http/schema"; export interface PluginSource { source: "github" | "url" | "git-subdir" | "archive"; @@ -25,6 +25,8 @@ export type SkillRegisterRequest = Omit; + export type SkillHubSkill = components["schemas"]["SkillHubSkill"]; export type SkillMetadata = components["schemas"]["SkillHubMetadata"]; export type SkillImport = Omit & diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index ef7c7511276..1ffe8f6eaa9 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -625,7 +625,7 @@ describe("Sidebar (leftnav)", () => { renderWithProviders(); - expect(screen.getByRole("link", { name: "Skills" })).toHaveAttribute("href", "/ui/skills"); + expect(screen.getByRole("link", { name: "Skills Hub" })).toHaveAttribute("href", "/ui/skills"); }); it.each([ diff --git a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx index 3fedeead150..e966836fa3a 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/PublicAiHub.integration.test.tsx @@ -82,6 +82,7 @@ const MODELS = [ const SKILLS: Plugin[] = [ { + download_count: 5, id: "skill-research", name: "research-assistant", description: "Answers research questions", @@ -92,11 +93,15 @@ const SKILLS: Plugin[] = [ category: "Analysis", keywords: ["evidence"], author: { name: "Research maintainers" }, + created_by: "writer@example.test", + approved_by: "reviewer@example.test", + security_scan_passed: true, enabled: true, created_at: null, updated_at: null, }, { + download_count: 9, id: "skill-review", name: "review-changes", description: "Checks pull requests", @@ -111,6 +116,7 @@ const SKILLS: Plugin[] = [ updated_at: null, }, { + download_count: 2, id: "skill-release", name: "release-notes", description: "Summarizes shipped changes", @@ -522,18 +528,18 @@ describe("PublicAiHub", () => { expect(screen.getByRole("link", { name: /release-notes/ })).toBeInTheDocument(); expect(screen.getByText("2 skills")).toBeInTheDocument(); - fireEvent.change(screen.getByRole("textbox", { name: "Search skills" }), { target: { value: "changelog" } }); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { target: { value: "changelog" } }); expect(screen.getByRole("link", { name: /release-notes/ })).toBeInTheDocument(); expect(screen.queryByRole("link", { name: /review-changes/ })).not.toBeInTheDocument(); - fireEvent.change(screen.getByRole("textbox", { name: "Search skills" }), { target: { value: "evidence" } }); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { target: { value: "evidence" } }); expect(screen.getByText('No skills match "evidence"')).toBeInTheDocument(); await user.click(screen.getByRole("button", { name: /^All domains/ })); expect(screen.getByRole("link", { name: /research-assistant/ })).toBeInTheDocument(); expect(screen.queryByRole("link", { name: /release-notes/ })).not.toBeInTheDocument(); - await user.click(screen.getByRole("button", { name: "Clear all" })); + await user.click(screen.getByRole("button", { name: "Clear filters" })); expect(screen.getByText("3 skills")).toBeInTheDocument(); - expect(screen.getByRole("textbox", { name: "Search skills" })).toHaveValue(""); + expect(screen.getByRole("textbox", { name: /Search skills/ })).toHaveValue(""); expect(screen.getByRole("checkbox", { name: "Engineering" })).not.toBeChecked(); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); }); @@ -541,7 +547,7 @@ describe("PublicAiHub", () => { it("finds skills by name, description, namespace, domain, and keywords", async () => { renderWithProviders(, { searchParams: { tab: "skills" } }); await screen.findByRole("link", { name: /research-assistant/ }); - const search = screen.getByRole("textbox", { name: "Search skills" }); + const search = screen.getByRole("textbox", { name: /Search skills/ }); for (const value of ["research-assistant", "Answers research questions", "knowledge", "Research", "evidence"]) { fireEvent.change(search, { target: { value } }); @@ -586,3 +592,34 @@ describe("PublicAiHub", () => { expect(screen.queryByRole("heading", { level: 2, name: "release-notes" })).not.toBeInTheDocument(); }); }); + +it("filters skills by tag and scan status, searches people, and scopes filters to Skills", async () => { + const user = userEvent.setup(); + const onUrlUpdate = vi.fn(); + renderWithProviders(, { searchParams: { tab: "skills" }, onUrlUpdate }); + await screen.findByRole("link", { name: /research-assistant/ }); + await user.click(screen.getByRole("button", { name: "Downloads" })); + expect(within(screen.getAllByRole("row")[1]).getByRole("link", { name: /review-changes/ })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Downloads" })); + expect(within(screen.getAllByRole("row")[1]).getByRole("link", { name: /release-notes/ })).toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by tag" }), "evidence"); + expect(screen.queryByRole("link", { name: /review-changes/ })).not.toBeInTheDocument(); + expect(screen.getByText("writer@example.test")).toBeInTheDocument(); + expect(screen.getByText("reviewer@example.test")).toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Security scan" }), "Security scan: No"); + expect(screen.queryByRole("link", { name: /research-assistant/ })).not.toBeInTheDocument(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Security scan" }), "Security scan: Yes"); + fireEvent.change(screen.getByRole("textbox", { name: /Search skills/ }), { + target: { value: "reviewer@example.test" }, + }); + expect(await screen.findByRole("link", { name: /research-assistant/ })).toBeInTheDocument(); + await waitFor(() => + expect(onUrlUpdate).toHaveBeenLastCalledWith( + expect.objectContaining({ + queryString: expect.stringContaining("skills_filter_tag=evidence"), + }), + ), + ); + await user.click(screen.getByRole("tab", { name: "Model Hub" })); + expect(screen.queryByRole("combobox", { name: "Filter by tag" })).not.toBeInTheDocument(); +}); diff --git a/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx b/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx index 4827a6fcc35..bcfade1849f 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/ResourcePanes.tsx @@ -5,17 +5,18 @@ import { Puzzle } from "lucide-react"; import * as React from "react"; import { useTranslation } from "react-i18next"; -import { getSourceDisplayText } from "@/components/claude_code_plugins/helpers"; import type { Plugin } from "@/components/claude_code_plugins/types"; +import { DataTableSortHeader } from "@/components/shared/DataTable/DataTableSortHeader"; import { DataTable } from "@/components/shared/DataTable"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; -import { matchesSearchTerm } from "@/utils/searchUtils"; +import { SkillFilters } from "@/components/claude_code_plugins/SkillFilters"; +import { getSkillMetadataColumns } from "@/components/claude_code_plugins/SkillMetadataColumns"; +import { useSkillFilters, matchesSkillFilters, skillTags } from "@/components/claude_code_plugins/skillCatalog"; import { CatalogDetails } from "./CatalogDetails"; import { CatalogFilters } from "./CatalogFilters"; import { LoadFailed, NoResults, NotListed } from "./HubEmpty"; -import { SearchInput } from "./HubParts"; import { SkillInspector } from "./SkillInspector"; import { useOpenHref } from "./useHubUrlState"; @@ -63,7 +64,8 @@ function SkillLink({ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect }: SkillsPaneProps) { const { t } = useTranslation(); - const [query, setQuery] = React.useState(""); + const filters = useSkillFilters(); + const query = filters.search; const [domains, setDomains] = React.useState([]); const skills = React.useMemo(() => [...(rows ?? [])].sort((a, b) => a.name.localeCompare(b.name)), [rows]); const options = React.useMemo( @@ -77,65 +79,34 @@ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect } const shown = React.useMemo( () => skills.filter((skill) => { - const matchesQuery = matchesSearchTerm(query, [ - skill.name, - skill.description, - skill.domain, - skill.namespace, - ...(skill.keywords ?? []), - ]); + const matchesQuery = matchesSkillFilters(skill, { + search: filters.search, + tag: filters.tag, + scan: filters.scan, + }); const matchesDomain = domains.length === 0 || domains.includes(skill.domain ?? ""); return matchesQuery && matchesDomain; }), - [skills, query, domains], + [skills, filters.search, filters.tag, filters.scan, domains], ); const opened = skills.find((skill) => skill.name === selected); const missingName = opened === undefined && !loading ? selected : null; const showRefreshError = error && !loading && shown.length > 0; - const filtered = query.trim() !== "" || domains.length > 0; + const filtered = filters.active || domains.length > 0; const clear = () => { - setQuery(""); + filters.clear(); setDomains([]); }; const columns: ColumnDef[] = [ { accessorKey: "name", - header: t("publicModelHub.catalog.skill", { defaultValue: "Skill" }), + header: ({ column }) => ( + + ), meta: { skeleton: "twoLine", className: "min-w-48 whitespace-normal" }, cell: ({ row }) => , }, - { - id: "source", - header: t("publicModelHub.catalog.source", { defaultValue: "Source" }), - meta: { - className: "hidden max-w-64 text-xs whitespace-normal wrap-anywhere text-muted-foreground lg:table-cell", - }, - cell: ({ row }) => getSourceDisplayText(row.original.source), - }, - { - accessorKey: "namespace", - header: t("publicModelHub.catalog.namespace", { defaultValue: "Namespace" }), - meta: { - className: - "hidden max-w-40 font-mono text-xs whitespace-normal wrap-anywhere text-muted-foreground xl:table-cell", - }, - cell: ({ row }) => row.original.namespace || "-", - }, - { - accessorKey: "category", - header: t("publicModelHub.catalog.category", { defaultValue: "Category" }), - meta: { className: "max-w-40 text-xs whitespace-normal wrap-anywhere text-muted-foreground" }, - cell: ({ row }) => row.original.category || "-", - }, - { - accessorKey: "version", - header: t("publicModelHub.catalog.version", { defaultValue: "Version" }), - meta: { - className: - "hidden max-w-32 font-mono text-xs whitespace-normal wrap-anywhere text-muted-foreground sm:table-cell", - }, - cell: ({ row }) => row.original.version || "-", - }, + ...getSkillMetadataColumns(), ]; const countOptions = { count: shown.length, @@ -183,16 +154,7 @@ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect }
- - {filtered && ( - - )} +
{loading @@ -203,6 +165,9 @@ export function SkillsPane({ rows, loading, error, onRetry, selected, onSelect }
skill.name} isLoading={loading} diff --git a/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx b/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx index 121d352565a..df94b2ea4f8 100644 --- a/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx +++ b/ui/litellm-dashboard/src/components/publicModelHub/SkillInspector.tsx @@ -1,5 +1,6 @@ "use client"; +import { DateCell } from "@/components/shared/table_cells"; import { ExternalLink } from "lucide-react"; import { useTranslation } from "react-i18next"; @@ -32,8 +33,6 @@ function sourceValue(source: Plugin["source"]) { export function SkillInspector({ skill }: { skill: Plugin }) { const { t } = useTranslation(); const keywords = skill.keywords ?? []; - const addedAt = skill.created_at ? new Date(skill.created_at) : null; - const added = addedAt && !Number.isNaN(addedAt.getTime()) ? addedAt.toLocaleDateString() : null; return (
}, + { label: t("skills.table.columns.updatedAt"), value: }, + { label: t("skills.table.columns.createdBy"), value: skill.created_by || t("skills.table.notRecorded") }, + { + label: t("skills.table.columns.publishedBy"), + value: skill.published_by || t("skills.table.notRecorded"), + }, + { label: t("skills.table.columns.approvedBy"), value: skill.approved_by || t("skills.table.notRecorded") }, + { + label: t("skills.table.columns.securityScan"), + value: ( + + {t(skill.security_scan_passed ? "skills.table.publicYes" : "skills.table.publicNo")} + + ), + }, ], [ { diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index ec67f4dc09c..f8886610e01 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -1693,9 +1693,9 @@ "unavailable": "An install command is unavailable for this gateway URL. Authenticated installs require HTTPS or a localhost gateway" }, "panel": { - "title": "Skills", - "intro": "Upload or import portable skills. Review proposed changes before publishing them to the Skills Hub.", - "addButton": "+ Add Skill", + "title": "Skills Hub", + "intro": "Register and share reusable skills for Codex, Claude Code, and other agents that support the open Agent Skills format.", + "addButton": "+ Register your skill", "deleteDialog": { "title": "Delete Skill", "confirm": "Are you sure you want to delete skill: ?", @@ -1718,8 +1718,8 @@ }, "table": { "empty": { - "title": "No skills found", - "description": "Add one to get started." + "title": "No skills registered yet", + "description": "Register your first skill to get started." }, "loading": "Loading skills…", "uncategorized": "Uncategorized", @@ -1739,15 +1739,22 @@ "description": "Description", "category": "Category", "public": "Public", - "createdAt": "Created At", + "createdAt": "Created", "actions": "Actions", "domain": "Domain", - "updatedAt": "Updated at", - "downloads": "Downloads" - } + "updatedAt": "Last updated", + "downloads": "Downloads", + "tags": "Tags", + "createdBy": "Created by", + "publishedBy": "Published by", + "approvedBy": "Approved by", + "securityScan": "Security scan" + }, + "notRecorded": "Not recorded", + "scanHelp": "Yes means this published version has a completed scan with no findings and a risk score of 0. No means it has not passed a recorded scan." }, "addForm": { - "title": "Add New Skill", + "title": "Register your skill", "sourceOptions": { "url": "Source URL", "upload": "Upload ZIP" @@ -1764,22 +1771,22 @@ "namespace": "Namespace (Optional)", "description": "Description (Optional)", "category": "Category (Optional)", - "keywords": "Keywords (Optional)", + "keywords": "Tags (Optional)", "version": "Version (Optional)", "authorName": "Author Name (Optional)", "authorEmail": "Author Email (Optional)" }, "hints": { "archive": "Choose a ZIP from your computer, up to 10 MiB. Include SKILL.md at the root or inside one skill folder, along with any supporting files. The skill will use the same visibility settings as skills added by URL.", - "sourceUrl": "Paste an HTTPS git repository URL from GitHub, GitLab, Bitbucket, or a self-hosted host (e.g. github.com/org/repo or github.com/org/repo/tree/main/my-skill), or an HTTPS link to a .zip archive of the skill hosted on S3 or any static file server. For a private repository use its SSH clone URL (git@ghe.example.com:org/repo.git) so Claude Code clones it with your own SSH key.", - "subPath": "Path within the repository where the skill lives (e.g., plugins/my-skill). Leave empty if the skill is at the repo root.", - "sha256": "Hex digest of the zip file. Claude Code refuses to install the archive if its checksum does not match.", + "sourceUrl": "Link to a Git repository, a skill folder, or a ZIP archive containing SKILL.md and any supporting files. For private repositories, use an SSH clone URL with access configured in your agent.", + "subPath": "Path within the repository where the skill lives (e.g., skills/my-skill). Leave empty if the skill is at the repo root.", + "sha256": "Optional SHA-256 digest for verifying the ZIP archive’s integrity.", "name": "Unique identifier in kebab-case format (e.g., my-skill)", - "domain": "Top-level grouping in the Skill Hub (e.g., Productivity)", + "domain": "Top-level grouping in the Skills Hub (e.g., Productivity)", "namespace": "Sub-grouping within domain (e.g., workflows)", "description": "Brief description of what the skill does", "category": "Select a category or enter a custom one", - "keywords": "Comma-separated list of keywords for search", + "keywords": "Comma-separated list of tags for search", "version": "Semantic version (e.g., 1.0.0)", "authorName": "Name of the skill author or organization", "authorEmail": "Contact email for the skill author" @@ -1800,13 +1807,13 @@ "detected": "Detected: {{source}}", "noCategories": "No matching categories", "cancel": "Cancel", - "adding": "Adding...", - "submit": "Add Skill", + "adding": "Registering...", + "submit": "Register skill", "errors": { "zipRequired": "Please choose a .zip file", "zipTooLarge": "The ZIP file must be 10 MiB or smaller", "urlRequired": "Please enter a repository or zip archive URL", - "subPathInvalid": "Subfolder must be a relative path like plugins/my-skill (letters, numbers, dots, hyphens, underscores)", + "subPathInvalid": "Subfolder must be a relative path like skills/my-skill (letters, numbers, dots, hyphens, underscores)", "sha256Invalid": "SHA-256 must be a 64-character hex digest", "nameRequired": "Please enter skill name", "nameInvalid": "Name must be kebab-case (lowercase, numbers, hyphens only)", @@ -1844,7 +1851,7 @@ "path": "Path (optional)", "domain": "Domain (optional)", "namespace": "Namespace (optional)", - "keywords": "Keywords (optional)" + "keywords": "Tags (optional)" }, "uploadHint": "Choose a ZIP up to 10 MiB containing SKILL.md. Its frontmatter supplies the skill name and description.", "replacementHint": "The replacement SKILL.md supplies the description. Domain, namespace, and keywords are preserved.", @@ -1921,7 +1928,7 @@ "source": "Source", "author": "Author", "homepage": "Homepage", - "keywords": "Keywords", + "keywords": "Tags", "category": "Category", "domain": "Domain", "namespace": "Namespace", @@ -1934,6 +1941,15 @@ "detail": { "subtitle": "Published skill metadata and portable archive", "private": "Private" + }, + "filters": { + "allTags": "All tags", + "tag": "Filter by tag", + "allScans": "All scan statuses", + "scanned": "Security scan: Yes", + "notScanned": "Security scan: No", + "search": "Search skills, tags, or people…", + "clear": "Clear filters" } }, "claudeCodePlugins": { @@ -1962,7 +1978,7 @@ "newer": "Newer", "older": "Older" }, - "back": "Skills", + "back": "Skills Hub", "tabs": { "overview": "Overview", "usage": "How to Use", @@ -2016,14 +2032,14 @@ "copy": "Copy", "copied": "Copied", "usage": { - "title": "Using this skill", + "title": "Use with Claude Code", "intro": "Once your proxy is set as a marketplace, enable this skill in Claude Code with one command:", "staleCatalog": "If you see \"Plugin {{name}} not found in marketplace\", update the catalog first:", "notConfigured": "Don't have the marketplace configured yet?", "seeSetup": "See one-time setup →" }, "setup": { - "title": "One-time marketplace setup", + "title": "Claude Code marketplace setup", "removeFirst": "If you added this marketplace before without a key, remove it first in Claude Code. Removing it also uninstalls the plugins you installed from it, so install them again from /plugin after you add the snippet:", "requiresKey": "This proxy requires a LiteLLM key to read the marketplace. Add this to and replace with your key:", "runCommand": "Run this command in Claude Code to register the marketplace:", @@ -2031,7 +2047,7 @@ } }, "makePublic": { - "title": "Publish to Skill Hub", + "title": "Publish to Skills Hub", "steps": { "select": "Select Skills", "confirm": "Confirm" @@ -2039,15 +2055,15 @@ "step1": { "title": "Select Skills to Publish", "selectAll": "Select All ({{total}})", - "description": "Selected skills will be visible to all users in the Skill Hub. Deselected skills will be unpublished.", + "description": "Selected skills will be visible to all users in the Skills Hub. Deselected skills will be unpublished.", "empty": "No skills registered yet.", "public": "Public", "selectedCount_one": "{{count}} skill will be published", "selectedCount_other": "{{count}} skills will be published" }, "step2": { - "title": "Confirm Publish to Skill Hub", - "note": "Note: Published skills will be visible to all users in the Skill Hub tab. Skills not in the list below will be unpublished.", + "title": "Confirm Publish to Skills Hub", + "note": "Note: Published skills will be visible to all users in the Skills Hub tab. Skills not in the list below will be unpublished.", "listTitle": "Skills to be published:", "total_one": "Total: {{count}} skill will be published", "total_other": "Total: {{count}} skills will be published" @@ -2060,8 +2076,8 @@ }, "toasts": { "selectAtLeastOne": "Please select at least one skill", - "published_one": "Skill Hub updated — {{count}} skill(s) published", - "published_other": "Skill Hub updated — {{count}} skill(s) published", + "published_one": "Skills Hub updated — {{count}} skill(s) published", + "published_other": "Skills Hub updated — {{count}} skill(s) published", "publishFailed": "Failed to update skills. Please try again." } } @@ -7102,7 +7118,7 @@ "models": "Model Hub", "agents": "Agent Hub", "mcp": "MCP Hub", - "skills": "Skill Hub" + "skills": "Skills Hub" }, "models": { "title": "Available Models", @@ -7265,7 +7281,7 @@ "models": "Model Hub", "agents": "Agent Hub", "mcp": "MCP Hub", - "skills": "Skill Hub" + "skills": "Skills Hub" }, "models": { "chatTitle": "Models", @@ -7688,7 +7704,7 @@ "agents": "Agents", "mcpShort": "MCP", "mcpServers": "MCP servers", - "skills": "Skills" + "skills": "Skills Hub" }, "header": { "title": "AI Hub", @@ -7954,7 +7970,7 @@ "workflowRuns": "Workflow Runs", "memory": "Memory", "mcpServers": "MCP Servers", - "skills": "Skills", + "skills": "Skills Hub", "guardrails": "Guardrails", "policies": "Policies", "tools": "Tools", @@ -8029,7 +8045,7 @@ "uiTheme": "Customize dashboard appearance", "tagManagement": "Organize resources with tags", "prompts": "Manage and version prompt templates", - "skills": "Browse and manage Claude Code skills", + "skills": "Register and share reusable agent skills in the Skills Hub", "routerSettings": "Configure routing and load balancing settings", "telemetry": "Choose which anonymous usage telemetry this proxy collects and see the last report", "loggingAndAlerts": "Set up logging and alert configurations", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a9af3f2b719..f9dae53bd03 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -42114,15 +42114,21 @@ export interface components { * @description Plugin item in list responses. */ PluginListItem: { + /** Approved By */ + approved_by?: string | null; author?: components["schemas"]["PluginAuthor"] | null; /** Category */ category?: string | null; /** Created At */ created_at: string | null; + /** Created By */ + created_by?: string | null; /** Description */ description: string | null; /** Domain */ domain?: string | null; + /** Download Count */ + download_count?: number | null; /** Enabled */ enabled: boolean; /** Homepage */ @@ -42137,6 +42143,13 @@ export interface components { name: string; /** Namespace */ namespace?: string | null; + /** Published By */ + published_by?: string | null; + /** + * Security Scan Passed + * @default false + */ + security_scan_passed: boolean; /** Source */ source: { [key: string]: string; @@ -46920,6 +46933,11 @@ export interface components { next_offset: number | null; /** Skills */ skills: components["schemas"]["SkillHubSkill"][]; + /** + * Tags + * @default [] + */ + tags: string[]; }; /** SkillHubMetadata */ SkillHubMetadata: { @@ -47005,15 +47023,21 @@ export interface components { }; /** SkillHubSkill */ SkillHubSkill: { + /** Approved By */ + approved_by?: string | null; /** Content Url */ content_url?: string | null; + /** Created At */ + created_at?: string | null; + /** Created By */ + created_by?: string | null; /** Description */ description?: string | null; /** Domain */ domain?: string | null; /** * Download Count - * @description Gateway ZIP downloads, included in proxy-admin catalog listings + * @description Gateway ZIP downloads, including repeat downloads */ download_count?: number | null; /** Download Url */ @@ -47026,6 +47050,13 @@ export interface components { name: string; /** Namespace */ namespace?: string | null; + /** Published By */ + published_by?: string | null; + /** + * Security Scan Passed + * @default false + */ + security_scan_passed: boolean; /** Source */ source: { [key: string]: string; @@ -74391,6 +74422,10 @@ export interface operations { query?: string; offset?: number; limit?: number; + tag?: string; + security_scan_passed?: boolean | null; + sort_by?: "name" | "download_count" | "updated_at" | "created_at" | "created_by" | "published_by" | "approved_by" | "security_scan_passed"; + sort_order?: "asc" | "desc"; }; header?: never; path?: never; From 9984ed40a18660c447d2eea88a91eb2ddfbd3454 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 5 Nov 2026 17:03:33 -0800 Subject: [PATCH 24/34] fix(ui): center the deployment health banner icon on its title (#50302) The banner inherits the Alert primitive's icon layout, which spans two grid rows and nudges the icon down 2px so it lines up with a title plus description. The banner only has a one-line title, so the empty second row and the nudge left the icon 3px below the text. Keep the icon on the title's row with no offset --- .../src/components/DeploymentHealth/DeploymentHealth.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentHealth.tsx b/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentHealth.tsx index 735a995d1e5..d513a4370d6 100644 --- a/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentHealth.tsx +++ b/ui/litellm-dashboard/src/components/DeploymentHealth/DeploymentHealth.tsx @@ -94,7 +94,7 @@ export const DeploymentHealthBanner = ({ handle }: { handle: DeploymentHealthHan variant={critical ? "error" : "warning"} role={critical ? "alert" : "status"} aria-label={t("shell.deploymentHealth.banner.label")} - className="min-h-10 shrink-0 items-center *:[svg]:text-warning data-[variant=error]:*:[svg]:text-destructive rounded-none border-x-0 border-t-0 border-b-warning/25 bg-warning/10 px-4 py-2 text-[13px] leading-5 text-foreground data-[variant=error]:border-b-destructive/25 data-[variant=error]:bg-destructive/10" + className="min-h-10 shrink-0 items-center *:[svg]:row-span-1 *:[svg]:translate-y-0 *:[svg]:text-warning data-[variant=error]:*:[svg]:text-destructive rounded-none border-x-0 border-t-0 border-b-warning/25 bg-warning/10 px-4 py-2 text-[13px] leading-5 text-foreground data-[variant=error]:border-b-destructive/25 data-[variant=error]:bg-destructive/10" > From fb2a1450dd7a1e88666379386b6e40883917113e Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Thu, 5 Nov 2026 17:07:19 -0800 Subject: [PATCH 25/34] fix(users)!: validate custom IDs and reject concurrent duplicate creation (#49748) * fix(users): validate custom IDs and reject concurrent duplicate creation * fix(users): return precise conflicts without retrying them * fix(users): preserve retries for existing-user upserts * fix(scim): preserve opaque IDs during user provisioning * fix(sso): preserve identity provider user IDs --- .../key_management_endpoints.py | 4 +- .../management_endpoints/scim/scim_v2.py | 4 +- litellm/proxy/management_endpoints/ui_sso.py | 8 +-- litellm/proxy/utils.py | 31 +++++++- litellm/repositories/user_repository.py | 6 +- .../internal_user_endpoints.py | 1 + .../proxy/management_endpoints/scim_v2.py | 9 +++ .../proxy/management_endpoints/ui_sso.py | 9 ++- .../management_endpoints/user_requests.py | 14 +++- .../scim/test_scim_v2_endpoints.py | 14 ++-- .../test_internal_user_endpoints.py | 50 ++++++++++++- .../proxy/management_endpoints/test_ui_sso.py | 70 +++++++++++++++++-- .../test_bulk_user_creation.py | 13 ++++ .../test_prisma_client_writes.py | 58 +++++++++++++++ 14 files changed, 267 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index c8895c01f7b..d28401524ed 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -5482,7 +5482,9 @@ async def generate_key_helper_fn( if query_type == "insert_data": user_row = cast( # cast-ok: table_name="user" is the insert_data branch returning the user row "prisma_models.LiteLLM_UserTable | None", - await prisma_client.insert_data(data=user_data, table_name="user"), + await prisma_client.insert_data( + data=user_data, table_name="user", allow_existing_user=request_type != "user" + ), ) if user_row is None: diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index e48f68a2d0a..66c2e54f5c6 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -1294,7 +1294,7 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou if litellm.default_internal_user_params: default_role = litellm.default_internal_user_params.get("user_role") - new_user_request: Final = NewUserRequest( + new_user_request: Final = SCIMNewUserRequest( user_id=user_id, user_email=user_id, # We don't have email from group membership user_alias=None, @@ -1873,7 +1873,7 @@ async def create_user( admin_group: Final = await _get_scim_admin_group() resolved_role: Final = _resolve_scim_user_role(user.groups or [], admin_group, default_role) - new_user_request: Final = NewUserRequest( + new_user_request: Final = SCIMNewUserRequest( user_id=user_id, user_email=user_data["user_email"], user_alias=user_data["user_alias"], diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 53079d84fdd..aaa261eb19b 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -78,7 +78,6 @@ from litellm.proxy._types import ( LitellmUserRoles, Member, NewTeamRequest, - NewUserRequest, NewUserRequestTeam, NewUserResponse, ProxyErrorTypes, @@ -164,6 +163,7 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( MicrosoftGraphAPIUserGroupResponse, MicrosoftServicePrincipalTeam, RoleMappings, + SSONewUserRequest, TeamMappings, ) from litellm.types.proxy.ui_sso import ParsedOpenIDResult @@ -2915,7 +2915,7 @@ async def insert_sso_user( if user_defined_values["user_role"] is None: user_defined_values["user_role"] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY - new_user_request: Final = NewUserRequest( + new_user_request: Final = SSONewUserRequest( user_id=user_defined_values["user_id"], user_email=normalize_email(user_defined_values["user_email"]), user_alias=_get_sso_user_alias(result_openid), @@ -2924,11 +2924,9 @@ async def insert_sso_user( budget_duration=user_defined_values["budget_duration"], sso_user_id=user_defined_values["user_id"], auto_create_key=False, + metadata={"auth_provider": getattr(result_openid, "provider")} if hasattr(result_openid, "provider") else {}, ) - if result_openid and hasattr(result_openid, "provider"): - new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")} - from litellm.proxy.proxy_server import premium_user with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True): diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 00187790901..3102674376a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -313,6 +313,7 @@ if TYPE_CHECKING: from prisma import models as prisma_models from prisma.actions import LiteLLM_DeprecatedVerificationTokenActions from prisma.client import TransactionManager + from prisma.errors import UniqueViolationError from prisma.models import LiteLLM_DeprecatedVerificationToken from prisma.types import HttpConfig, LiteLLM_VerificationTokenInclude @@ -5559,6 +5560,25 @@ _LOCK_TEAMS_BY_ID_SQL: Final = ( ) +def _user_unique_conflict_message(metadata: object, user_id: object) -> str: + match metadata: + case {"target": ["user_id"]}: + return f"User with id {user_id} already exists" + case {"target": ["sso_user_id"]}: + return "User with this sso_user_id already exists" + case _: + return "User conflicts with an existing unique value" + + +def _raise_user_unique_conflict(error: "UniqueViolationError", user_id: object, *, allow_existing: bool) -> NoReturn: + if allow_existing: + raise error + raise HTTPException( + status_code=409, + detail={"error": _user_unique_conflict_message(error.meta, user_id)}, + ) from error + + class PrismaClient: spend_log_transactions: list = [] _spend_log_transactions_lock = asyncio.Lock() @@ -6446,12 +6466,15 @@ class PrismaClient: max_tries=3, # maximum number of retries max_time=10, # maximum total time to retry for on_backoff=on_backoff, # specifying the function to call on backoff + giveup=lambda exc: isinstance(exc, HTTPException) and exc.status_code == 409, ) @log_db_metrics async def insert_data( self, data: Mapping[str, object], table_name: Literal["user", "key", "config", "spend", "team", "user_notification"], + *, + allow_existing_user: bool = True, ): """ Add a key to the database. If it already exists, do nothing. @@ -6487,14 +6510,20 @@ class PrismaClient: verbose_proxy_logger.info("Data Inserted into Keys Table") return new_verification_token elif table_name == "user": + from prisma.errors import UniqueViolationError + db_data = self.jsonify_object(data=data) try: - new_user_row: Final = await UserRepository(self).upsert_user(db_data) + new_user_row: Final = await UserRepository(self).upsert_user( + db_data, allow_existing=allow_existing_user + ) if new_user_row is None: raise HTTPException( status_code=403, detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.", ) + except UniqueViolationError as e: + _raise_user_unique_conflict(e, db_data["user_id"], allow_existing=allow_existing_user) except Exception as e: if ( "Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`" diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 1fce0717ea3..28fba2de564 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -168,7 +168,9 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): """ return await _count_billable_users(self.table) - async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None": + async def upsert_user( + self, data: Mapping[str, object], *, allow_existing: bool = True + ) -> "prisma_models.LiteLLM_UserTable | None": """Return the user row, or None when a new free SSO user would exceed capacity.""" scope: Final = _FREE_SSO_CREATION.get() if ( @@ -176,6 +178,8 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): or (scope[0] is not None and scope[0] != data["user_id"]) or scope[1] is not asyncio.current_task() ): + if not allow_existing: + return await self.table.create(data=dict(data)) return await self.table.upsert( where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} ) diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index fc0b73f77b6..e79b509da1b 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -120,6 +120,7 @@ class BulkNewUserItem(NewUserRequest): model_config = ConfigDict(extra="forbid", protected_namespaces=()) + user_id: str | None = None auto_create_key: bool = False @field_validator("send_invite_email") diff --git a/litellm/types/proxy/management_endpoints/scim_v2.py b/litellm/types/proxy/management_endpoints/scim_v2.py index fadb309e0cd..72a39c61967 100644 --- a/litellm/types/proxy/management_endpoints/scim_v2.py +++ b/litellm/types/proxy/management_endpoints/scim_v2.py @@ -13,6 +13,7 @@ from pydantic import ( from pydantic_core.core_schema import SerializerFunctionWrapHandler from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.management_endpoints.user_requests import NewUserRequest SCIM_ENTERPRISE_USER_SCHEMA: Final = "urn:ietf:params:scim:schemas:extension:enterprise:2.0:User" SCIM_ENTERPRISE_METADATA_KEY: Final = "scim_enterprise" @@ -26,6 +27,14 @@ _SCIM_SERIALIZED_FIELDS_ADAPTER: Final = TypeAdapter( ) +class SCIMNewUserRequest(NewUserRequest): + """Preserve opaque directory IDs when provisioning SCIM users and group members.""" + + model_config = ConfigDict(frozen=True) + + user_id: str | None = None + + class LiteLLM_UserScimMetadata(LiteLLMBaseModel): """ Scim metadata stored in LiteLLM_UserTable.metadata diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index affb6d2e9ce..38ed799f38b 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -1,12 +1,19 @@ from typing import Literal -from pydantic import Field +from pydantic import ConfigDict, Field from typing_extensions import TypedDict from litellm.types.proxy.auth.user_roles import KeyManagementRoutes, LitellmUserRoles +from litellm.types.proxy.management_endpoints.user_requests import NewUserRequest from litellm.types.utils import LiteLLMPydanticObjectBase +class SSONewUserRequest(NewUserRequest): + model_config = ConfigDict(frozen=True) + + user_id: str | None = None + + class LiteLLM_UpperboundKeyGenerateParams(LiteLLMPydanticObjectBase): """ Set default upperbound to max budget a key called via `/key/generate` can be. diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py index e53dc4152c3..edda1f78430 100644 --- a/litellm/types/proxy/management_endpoints/user_requests.py +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -1,7 +1,7 @@ from collections.abc import Mapping -from typing import Literal +from typing import Annotated, Final, Literal -from pydantic import Field, field_validator, model_validator +from pydantic import AfterValidator, Field, field_validator, model_validator from litellm.models.user import LiteLLM_UserTable from litellm.types.proxy.auth.user_roles import LitellmUserRoles @@ -15,7 +15,17 @@ class NewUserRequestTeam(LiteLLMPydanticObjectBase): user_role: Literal["user", "admin"] = "user" +def _validate_custom_user_id(value: str) -> str: + normalized: Final = value.strip() + if not normalized: + raise ValueError("user_id must not be empty or contain only whitespace") + if "\x00" in normalized: + raise ValueError("user_id must not contain NUL characters") + return normalized + + class NewUserRequest(GenerateRequestBase): + user_id: Annotated[str, AfterValidator(_validate_custom_user_id)] | None = None tpm_limit: int | None = Field(default=None, ge=0) rpm_limit: int | None = Field(default=None, ge=0) max_budget: float | None = None diff --git a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index 180362f45f1..94e0f91ecd1 100644 --- a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -411,10 +411,11 @@ async def test_create_user_without_groups_defers_to_default_team(mocker: MockerF @pytest.mark.asyncio -async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, monkeypatch): +@pytest.mark.parametrize("user_id", ["new-user", " employee-42 "]) +async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, monkeypatch, user_id: str): scim_user = SCIMUser( schemas=["urn:ietf:params:scim:schemas:core:2.0:User"], - userName="new-user", + userName=user_id, emails=[SCIMUserEmail(value="new@example.com")], groups=[SCIMUserGroup(value="idp-team")], ) @@ -428,10 +429,12 @@ async def test_create_user_with_groups_keeps_idp_teams(mocker: MockerFixture, mo await create_user(user=scim_user) assert new_user_mock.call_args.kwargs["data"].teams == ["idp-team"] + assert new_user_mock.call_args.kwargs["data"].user_id == user_id @pytest.mark.asyncio -async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFixture, monkeypatch): +@pytest.mark.parametrize("user_id", ["group-user", " employee-42 "]) +async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFixture, monkeypatch, user_id: str): monkeypatch.setattr( "litellm.default_internal_user_params", {"teams": [{"team_id": "default-team", "max_budget_in_team": 25}]}, @@ -439,14 +442,15 @@ async def test_create_user_if_not_exists_defers_to_default_team(mocker: MockerFi ) new_user_mock = mocker.patch( # test-quality-ok: new_user is imported inside the helper, not injectable "litellm.proxy.management_endpoints.internal_user_endpoints.new_user", - AsyncMock(return_value=NewUserResponse(user_id="group-user", key="k")), + AsyncMock(return_value=NewUserResponse(user_id=user_id, key="k")), ) - created = await _create_user_if_not_exists(user_id="group-user") + created = await _create_user_if_not_exists(user_id=user_id) assert created is not None assert new_user_mock.call_args.kwargs["data"].teams is None assert new_user_mock.call_args.kwargs["user_api_key_dict"] == UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN) + assert new_user_mock.call_args.kwargs["data"].user_id == user_id @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index cee0c90f80c..b96a5b606eb 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -13,6 +13,7 @@ import httpx import pytest from fastapi import HTTPException from fastapi.testclient import TestClient +from pydantic import ValidationError from pytest_mock import MockerFixture from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler @@ -48,6 +49,51 @@ from tests.unit.proxy.management_endpoints.jwt_key_mapping_doubles import ( client = TestClient(app) + +@pytest.mark.parametrize("user_id", ["", " \t\n", "user\x00id", 123, True, [], {}]) +def test_new_user_rejects_invalid_custom_id(user_id: object) -> None: + with pytest.raises(ValidationError) as exc: + NewUserRequest.model_validate({"user_id": user_id}) + assert exc.value.errors()[0]["loc"] == ("user_id",) + + +@pytest.mark.parametrize("user_id", [None, "employee-42", "社員@example.test", " employee-42 "]) +def test_new_user_normalizes_custom_id_and_preserves_auto_generation(user_id: str | None) -> None: + request: Final = NewUserRequest(user_id=user_id) + assert request.user_id == (user_id.strip() if user_id is not None else None) + + +@pytest.mark.asyncio +async def test_new_user_returns_conflict_when_id_is_taken_after_duplicate_check(mocker: MockerFixture) -> None: + from prisma.errors import UniqueViolationError + + from litellm.proxy import proxy_server + from litellm.proxy.utils import PrismaClient + + table: Final = SimpleNamespace( + find_first=AsyncMock(return_value=None), + count=AsyncMock(return_value=0), + create=AsyncMock(side_effect=UniqueViolationError( + {"user_facing_error": {"meta": {"target": ["user_id"]}}}, + message="Unique constraint failed on user_id", + )), + upsert=AsyncMock(return_value=LiteLLM_UserTable(user_id="employee-42")), + ) + mocker.patch("prisma.Prisma", return_value=SimpleNamespace(litellm_usertable=table)) + database: Final = PrismaClient( + database_url="postgresql://unused:unused@localhost/unused", + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + mocker.patch("litellm.proxy.proxy_server.prisma_client", database) + with pytest.raises(ProxyException) as exc: + await new_user( + data=NewUserRequest(user_id="employee-42", auto_create_key=False, send_invite_email=False), + user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert str(exc.value.code) == "409" + assert "employee-42" in exc.value.message + + if TYPE_CHECKING: from litellm.caching.redis_cache import RedisCache from litellm.proxy.utils import PrismaClient @@ -4862,7 +4908,9 @@ async def test_user_new_persists_model_max_budget(monkeypatch, model_max_budget, models = [] class _FakePrisma: - async def insert_data(self, data, table_name): + async def insert_data( + self, data: Mapping[str, object], table_name: str, *, allow_existing_user: bool = True + ) -> _FakeUserRow | SimpleNamespace: if table_name == "user": captured["user_data"] = dict(data) return _FakeUserRow() diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index ba78fefd653..6ccb019b937 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -22,6 +22,7 @@ from litellm.proxy._types import ( LiteLLM_JWTAuth, LiteLLM_UserTable, NewUserResponse, + SSOUserDefinedValues, ProxyException, ) from litellm.proxy.auth.handle_jwt import JWTHandler @@ -52,6 +53,62 @@ from litellm.types.proxy.management_endpoints.ui_sso import ( from tests._master_key import MASTER_KEY +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("billable_users", "premium", "admitted"), + [(4, False, True), (5, False, False), (26, False, False), (26, True, True)], +) +async def test_sso_auto_creation_enforces_free_user_cap_through_new_user( + monkeypatch: pytest.MonkeyPatch, billable_users: int, premium: bool, admitted: bool +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.utils import PrismaClient + + row: Final = LiteLLM_UserTable(user_id="new-sso-user", user_email="new-sso-user@example.test") + table: Final = SimpleNamespace( + find_first=AsyncMock(return_value=None), + find_unique=AsyncMock(return_value=None), + count=AsyncMock(side_effect=lambda where=None: billable_users if where is None else 0), + create=AsyncMock(return_value=row), + upsert=AsyncMock(return_value=row), + ) + transaction: Final = MagicMock() + transaction.__aenter__ = AsyncMock( + return_value=SimpleNamespace(litellm_usertable=table, execute_raw=AsyncMock(), query_raw=AsyncMock()) + ) + transaction.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr( + "prisma.Prisma", + lambda **kwargs: SimpleNamespace(litellm_usertable=table, tx=MagicMock(return_value=transaction)), + ) + database: Final = PrismaClient( + database_url="postgresql://unused:unused@localhost/unused", + proxy_logging_obj=proxy_server.proxy_logging_obj, + ) + monkeypatch.setattr(proxy_server, "prisma_client", database) + monkeypatch.setattr(proxy_server, "premium_user", premium) + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + values: Final[SSOUserDefinedValues] = { + "models": [], + "user_id": row.user_id, + "user_email": row.user_email, + "user_role": "internal_user_viewer", + "max_budget": None, + "budget_duration": None, + } + if admitted: + response: Final = await insert_sso_user(CustomOpenID(id=row.user_id, team_ids=[]), values) + assert response.user_id == row.user_id + assert table.create.await_count + table.upsert.await_count == 1 + else: + with pytest.raises(ProxyException) as error: + await insert_sso_user(CustomOpenID(id=row.user_id, team_ids=[]), values) + assert str(error.value.code) == "403" + assert "SSO for more than 5 users" in error.value.message + table.create.assert_not_awaited() + table.upsert.assert_not_awaited() + + def _sso_http_handler(route: Callable[[httpx.Request], httpx.Response] | None = None) -> AsyncHTTPHandler: """litellm's SSO httpx client answered in-process, by ``route`` or by the active respx mock.""" handler = AsyncHTTPHandler() @@ -1281,7 +1338,8 @@ async def test_upsert_sso_user_fills_user_alias_for_existing_user(): @pytest.mark.asyncio -async def test_insert_sso_user_sets_user_alias_from_display_name(): +@pytest.mark.parametrize("user_id", ["S-1-5-21-adfs-user", " employee-42 "]) +async def test_insert_sso_user_sets_user_alias_from_display_name(user_id: str): """ A newly created SSO user is inserted with the IdP display name as user_alias. """ @@ -1290,7 +1348,7 @@ async def test_insert_sso_user_sets_user_alias_from_display_name(): from litellm.proxy.management_endpoints.ui_sso import insert_sso_user sso_result = CustomOpenID( - id="S-1-5-21-adfs-user", + id=user_id, email="jane.doe@example.com", first_name="Jane", last_name="Doe", @@ -1300,7 +1358,7 @@ async def test_insert_sso_user_sets_user_alias_from_display_name(): ) user_defined_values: SSOUserDefinedValues = { "models": [], - "user_id": "S-1-5-21-adfs-user", + "user_id": user_id, "user_email": "jane.doe@example.com", "max_budget": None, "user_role": "internal_user", @@ -1309,12 +1367,14 @@ async def test_insert_sso_user_sets_user_alias_from_display_name(): with patch( "litellm.proxy.management_endpoints.ui_sso.new_user", - return_value=NewUserResponse(user_id="S-1-5-21-adfs-user", key="sk-xxxxx", teams=None), + return_value=NewUserResponse(user_id=user_id, key="sk-xxxxx", teams=None), ) as mock_new_user: await insert_sso_user(result_openid=sso_result, user_defined_values=user_defined_values) new_user_request = mock_new_user.call_args.kwargs["data"] - assert new_user_request.user_id == "S-1-5-21-adfs-user" + assert new_user_request.user_id == user_id + assert new_user_request.sso_user_id == user_id + assert new_user_request.metadata == {"auth_provider": "generic"} assert new_user_request.user_email == "jane.doe@example.com" assert new_user_request.user_alias == "Doe, Jane" diff --git a/tests/unit/proxy/management_helpers/test_bulk_user_creation.py b/tests/unit/proxy/management_helpers/test_bulk_user_creation.py index cf1ac61c93a..2b966a8ebeb 100644 --- a/tests/unit/proxy/management_helpers/test_bulk_user_creation.py +++ b/tests/unit/proxy/management_helpers/test_bulk_user_creation.py @@ -191,6 +191,19 @@ async def _run(prisma, users, caller=ADMIN, license=None, generate_key=_no_keys) ) +@pytest.mark.asyncio +@pytest.mark.parametrize("user_id", [None, "", " employee-42 ", " \t"]) +async def test_bulk_keeps_existing_user_id_generation_and_whitespace_behavior(user_id: str | None) -> None: + prisma: Final = _FakePrisma() + response: Final = await _run(prisma, [{"user_id": user_id}]) + result: Final = response.data[0] + assert result.success + assert result.user_id + assert result.user_id in prisma.db.litellm_usertable.rows + if user_id: + assert result.user_id == user_id + + @pytest.mark.asyncio async def test_creates_users_and_team_membership_in_every_store(): prisma = _FakePrisma(teams=[_team("t1", [Member(user_id="existing", role="admin")]), _team("t2")]) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py index e335b08f040..8603de0c3a1 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py @@ -18,8 +18,10 @@ from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from fastapi import HTTPException +from prisma.errors import UniqueViolationError from litellm._service_logger import ServiceTypes from litellm.models.user import LiteLLM_UserTable @@ -192,6 +194,62 @@ async def test_insert_data_team_serializes_members(prisma_client: PrismaClient) assert create_payload["team_id"] == "t1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("metadata", "message"), + [ + ({"target": ["user_id"]}, "User with id requested-user already exists"), + ({"target": ["sso_user_id"]}, "User with this sso_user_id already exists"), + (None, "User conflicts with an existing unique value"), + ({"target": ["unknown_field"]}, "User conflicts with an existing unique value"), + ], +) +async def test_user_conflict_names_constraint_and_does_not_retry( + prisma_client: PrismaClient, + metadata: Mapping[str, object] | None, + message: str, +) -> None: + create: Final = AsyncMock(side_effect=UniqueViolationError({"user_facing_error": {"meta": metadata}})) + prisma_client.db.litellm_usertable.create = create + with pytest.raises(HTTPException) as error: + await prisma_client.insert_data( + data={"user_id": "requested-user", "sso_user_id": "sso-subject"}, + table_name="user", allow_existing_user=False, + ) + assert error.value.status_code == 409 + assert error.value.detail == {"error": message} + assert create.await_count == 1 + + +@pytest.mark.asyncio +async def test_user_upsert_recovers_from_concurrent_creation( + prisma_client: PrismaClient, monkeypatch: pytest.MonkeyPatch +) -> None: + row: Final = LiteLLM_UserTable(user_id="requested-user") + conflict: Final = UniqueViolationError({"user_facing_error": {"meta": {"target": ["user_id"]}}}) + prisma_client.db.litellm_usertable.upsert = AsyncMock(side_effect=[conflict, row]) + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + + result: Final = await prisma_client.insert_data(data={"user_id": row.user_id}, table_name="user") + + assert result is row + + +@pytest.mark.asyncio +async def test_user_insert_still_retries_transient_connection_failure( + prisma_client: PrismaClient, monkeypatch: pytest.MonkeyPatch +) -> None: + row: Final = LiteLLM_UserTable(user_id="requested-user") + create: Final = AsyncMock(side_effect=[httpx.ConnectError("connection unavailable"), row]) + prisma_client.db.litellm_usertable.create = create + monkeypatch.setattr("asyncio.sleep", AsyncMock()) + result: Final = await prisma_client.insert_data( + data={"user_id": row.user_id}, table_name="user", allow_existing_user=False + ) + assert result is row + assert create.await_count == 2 + + @pytest.mark.asyncio async def test_insert_data_user_organization_fk_raises_400( prisma_client: PrismaClient, From 7b550cdd94a38c3607647f1053c5164a076c27e0 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Thu, 5 Nov 2026 17:18:00 -0800 Subject: [PATCH 26/34] feat(ui): allow custom user IDs in user invitations (#49749) * feat(ui): allow custom user IDs in user invitations * test(ui): assert unexpected user-creation requests --- .../CreateUserButton.integration.test.tsx | 111 ++++++++++++++++++ .../src/components/CreateUserButton.test.tsx | 2 +- .../src/components/CreateUserButton.tsx | 57 ++++++++- 3 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx new file mode 100644 index 00000000000..e89211596f3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CreateUserButton.integration.test.tsx @@ -0,0 +1,111 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CreateUserButton } from "./CreateUserButton"; + +const createUser = vi.fn<(body: Record) => Response>(); +const unexpectedRequest = vi.fn((path: string) => + Response.json({ error: `Unexpected request ${path}` }, { status: 500 }), +); + +beforeEach(() => { + unexpectedRequest.mockClear(); + createUser.mockReset(); + createUser.mockImplementation((body) => Response.json({ user_id: body.user_id ?? "generated-id" })); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const path = new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname; + if (path === "/user/new") return createUser(JSON.parse(String(init?.body))); + if (path === "/models") return Response.json({ data: [] }); + if (path === "/sso/get/ui_settings") return Response.json({ SSO_ENABLED: true }); + if (path === "/v2/team/list") return Response.json({ teams: [], total: 0, total_pages: 0 }); + return unexpectedRequest(path); + }), + ); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + expect(unexpectedRequest).not.toHaveBeenCalled(); +}); + +const renderForm = (isEmbedded: boolean, onUserCreated = vi.fn()) => { + const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + render( + + + , + ); + return onUserCreated; +}; + +describe("custom User ID invitation", () => { + it.each([true, false])("sends the custom ID from the form (embedded=%s)", async (embedded) => { + const user = userEvent.setup(); + const onCreated = renderForm(embedded); + if (!embedded) await user.click(screen.getByRole("button", { name: "+ Invite User" })); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { + target: { value: " employee-42 " }, + }); + fireEvent.change(screen.getByRole("textbox", { name: "User Email" }), { + target: { value: "employee@example.test" }, + }); + await user.click(screen.getByRole("button", { name: embedded ? "Create User" : "Invite User", exact: true })); + await waitFor(() => + expect(createUser).toHaveBeenCalledWith( + expect.objectContaining({ + user_id: "employee-42", + user_email: "employee@example.test", + auto_create_key: false, + }), + ), + ); + expect(createUser.mock.calls[0][0]).not.toHaveProperty("supply_user_id"); + if (embedded) { + await waitFor(() => expect(onCreated).toHaveBeenCalledWith("employee-42")); + expect(screen.getByRole("checkbox", { name: "Supply custom User ID" })).not.toBeChecked(); + expect(screen.queryByRole("textbox", { name: "User ID", exact: true })).not.toBeInTheDocument(); + } + }); + + it("blocks whitespace and clears the custom ID when automatic generation is selected", async () => { + const user = userEvent.setup(); + renderForm(true); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { target: { value: " " } }); + await user.click(screen.getByRole("button", { name: "Create User" })); + expect(await screen.findByText("Enter a User ID")).toBeInTheDocument(); + expect(createUser).not.toHaveBeenCalled(); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { + target: { value: "discard-me" }, + }); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + await user.click(screen.getByRole("button", { name: "Create User" })); + await waitFor(() => expect(createUser).toHaveBeenCalledWith(expect.objectContaining({ user_id: null }))); + expect(createUser.mock.calls[0][0]).not.toHaveProperty("supply_user_id"); + }); + + it("discards a custom ID when an invitation is cancelled", async () => { + const user = userEvent.setup(); + renderForm(false); + await user.click(screen.getByRole("button", { name: "+ Invite User" })); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + fireEvent.change(screen.getByRole("textbox", { name: "User ID", exact: true }), { target: { value: "cancelled" } }); + await user.click( + within(screen.getByRole("dialog", { name: "Invite User" })).getByRole("button", { name: "Close" }), + ); + await user.click(screen.getByRole("button", { name: "+ Invite User" })); + expect(screen.getByRole("checkbox", { name: "Supply custom User ID" })).not.toBeChecked(); + await user.click(screen.getByRole("checkbox", { name: "Supply custom User ID" })); + expect(screen.getByRole("textbox", { name: "User ID", exact: true })).toHaveValue(""); + }); +}); diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx index 94b3726ede0..d04272cecd3 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.test.tsx @@ -308,7 +308,7 @@ describe("CreateUserButton", () => { const dialog = screen.getByRole("dialog", { name: /invite user/i }); - expectControlBesideLabel(within(dialog).getByRole("checkbox")); + expectControlBesideLabel(within(dialog).getByRole("checkbox", { name: "Send invitation email" })); }); describe("organizations", () => { diff --git a/ui/litellm-dashboard/src/components/CreateUserButton.tsx b/ui/litellm-dashboard/src/components/CreateUserButton.tsx index 6fbe8f2d2bd..bf188d36716 100644 --- a/ui/litellm-dashboard/src/components/CreateUserButton.tsx +++ b/ui/litellm-dashboard/src/components/CreateUserButton.tsx @@ -14,7 +14,7 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp import { Alert, AlertDescription, AlertTitle } from "@/components/shared/Alert"; import { ChevronRight, CircleHelp, Info, UserPlus } from "lucide-react"; import React, { useEffect, useState } from "react"; -import { useForm } from "react-hook-form"; +import { useForm, useWatch } from "react-hook-form"; import { useTranslation } from "react-i18next"; import TeamDropdown from "./common_components/team_dropdown"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; @@ -58,6 +58,8 @@ interface UISettings { } interface CreateUserFormValues { + supply_user_id: boolean; + user_id?: string; user_email?: string; user_role: string; team_id?: string | null; @@ -68,6 +70,8 @@ interface CreateUserFormValues { } const EMBEDDED_DEFAULTS: CreateUserFormValues = { + supply_user_id: false, + user_id: "", user_email: undefined, user_role: "internal_user_viewer", team_id: undefined, @@ -76,6 +80,8 @@ const EMBEDDED_DEFAULTS: CreateUserFormValues = { }; const STANDALONE_DEFAULTS: CreateUserFormValues = { + supply_user_id: false, + user_id: "", user_email: undefined, user_role: "internal_user_viewer", team_id: undefined, @@ -95,11 +101,13 @@ const withoutCollapsedModels = (values: CreateUserFormValues, isPersonalKeyOpen: return rest; }; -const buildCreatePayload = (values: CreateUserFormValues): CreateUserFormValues & { organizations?: string[] } => { +const buildCreatePayload = (values: CreateUserFormValues) => { + const { supply_user_id, user_id, ...fields } = values; + const apiValues = supply_user_id ? { ...fields, user_id: user_id?.trim() } : fields; const withModels = (!values.models || values.models.length === 0) && values.user_role !== "proxy_admin" - ? { ...values, models: ["no-default-models"] } - : values; + ? { ...apiValues, models: ["no-default-models"] } + : apiValues; if (!withModels.organization_ids) { return withModels; } @@ -163,6 +171,7 @@ export const CreateUserButton: React.FC = ({ const [uiSettings, setUISettings] = useState(null); const defaultValues = isEmbedded ? EMBEDDED_DEFAULTS : STANDALONE_DEFAULTS; const form = useForm({ defaultValues }); + const supplyUserId = useWatch({ control: form.control, name: "supply_user_id" }); const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(false); const [userModels, setUserModels] = useState([]); @@ -205,6 +214,14 @@ export const CreateUserButton: React.FC = ({ }; const handleCreate = async (formValues: CreateUserFormValues) => { + if (formValues.supply_user_id && !formValues.user_id?.trim()) { + form.setError("user_id", { message: "Enter a User ID" }, { shouldFocus: true }); + return; + } + if (formValues.supply_user_id && formValues.user_id?.includes("\u0000")) { + form.setError("user_id", { message: "User ID must not contain NUL characters" }, { shouldFocus: true }); + return; + } try { toast.info(t("users.createUser.toasts.makingApiCall")); if (!isEmbedded) { @@ -255,6 +272,34 @@ export const CreateUserButton: React.FC = ({ ); + const userIdField = ( + <> + + {({ id, value, onChange, onBlur }) => ( + { + onChange(checked); + form.resetField("user_id"); + }} + onBlur={onBlur} + /> + )} + + {supplyUserId && ( + + {({ ref, value, ...control }) => } + + )} + + ); + const teamField = ( = ({ {userEmailField} + {userIdField} {roleField(t("users.createUser.fields.userRole"))} {teamField} {metadataField} @@ -357,6 +403,7 @@ export const CreateUserButton: React.FC = ({
{userEmailField} + {userIdField} {roleField( labelWithHint(t("users.createUser.fields.globalRole"), t("users.createUser.fields.globalRoleHint")), )} @@ -436,7 +483,7 @@ export const CreateUserButton: React.FC = ({ -
+
)} +
diff --git a/ui/litellm-dashboard/src/i18n/I18nProvider.test.tsx b/ui/litellm-dashboard/src/i18n/I18nProvider.test.tsx new file mode 100644 index 00000000000..1cf0c8971a0 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/I18nProvider.test.tsx @@ -0,0 +1,82 @@ +import { render, screen } from "@testing-library/react"; +import i18next, { type InitOptions } from "i18next"; +import { useState } from "react"; +import { I18nextProvider, initReactI18next, useTranslation } from "react-i18next"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { LanguageGate } from "./I18nProvider"; +import { Locales, Messages } from "./languages"; +import en from "./locales/en.json"; + +const JAPANESE_LOGOUT = "ログアウト"; +const japanese: Messages = { shell: { sidebarAccountMenu: { logout: JAPANESE_LOGOUT } } }; + +const options: InitOptions = { + lng: "en", + fallbackLng: "en", + interpolation: { escapeValue: false }, + resources: { en: { translation: en } }, + initAsync: false, +}; + +const createI18n = () => { + const instance = i18next.createInstance(); + void instance.use(initReactI18next).init(options); + return instance; +}; + +function AccountMenuLabels() { + const { t, i18n } = useTranslation(); + const [labelReadOnce] = useState(() => i18n.t("shell.sidebarAccountMenu.logout")); + return ( + <> +

{t("shell.sidebarAccountMenu.logout")}

+

{labelReadOnce}

+

{t("shell.sidebarAccountMenu.changePassword")}

+ + ); +} + +const renderGate = (stored: string | null, locales: Locales) => { + const instance = createI18n(); + render( + + stored, write: vi.fn() }}> + + + , + ); +}; + +afterEach(() => { + document.documentElement.lang = "en"; + vi.restoreAllMocks(); +}); + +describe("LanguageGate", () => { + it("shows the stored language, with English for strings it hasn't translated", async () => { + renderGate("ja", new Map([["ja", async () => japanese]])); + + expect(await screen.findByTestId("logout")).toHaveTextContent(JAPANESE_LOGOUT); + expect(screen.getByTestId("logout-read-once")).toHaveTextContent(JAPANESE_LOGOUT); + expect(screen.getByTestId("change-password")).toHaveTextContent(en.shell.sidebarAccountMenu.changePassword); + expect(document.documentElement.lang).toBe("ja"); + }); + + it.each([null, "en", "fr"])("shows English without loading anything when the stored value is %j", (stored) => { + const load = vi.fn(async () => japanese); + renderGate(stored, new Map([["ja", load]])); + + expect(screen.getByTestId("logout")).toHaveTextContent(en.shell.sidebarAccountMenu.logout); + expect(load).not.toHaveBeenCalled(); + expect(document.documentElement.lang).toBe("en"); + }); + + it("falls back to English when the translations fail to load", async () => { + vi.spyOn(console, "warn").mockImplementation(() => undefined); + renderGate("ja", new Map([["ja", () => Promise.reject(new Error("offline"))]])); + + expect(await screen.findByTestId("logout")).toHaveTextContent(en.shell.sidebarAccountMenu.logout); + expect(document.documentElement.lang).toBe("en"); + }); +}); diff --git a/ui/litellm-dashboard/src/i18n/I18nProvider.tsx b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx index f0ecdfe2568..f96658fbbb3 100644 --- a/ui/litellm-dashboard/src/i18n/I18nProvider.tsx +++ b/ui/litellm-dashboard/src/i18n/I18nProvider.tsx @@ -1,9 +1,55 @@ "use client"; +import type { i18n as I18n } from "i18next"; +import { useEffect, useState, useSyncExternalStore } from "react"; import { I18nextProvider } from "react-i18next"; +import LoadingScreen from "@/components/common_components/LoadingScreen"; + import i18n from "./i18n"; +import { browserLanguageStore, DEFAULT_LANGUAGE, LanguageStore, Locales, LOCALES, resolveLanguage } from "./languages"; + +type LanguageGateProps = { + i18n: I18n; + locales: Locales; + store: LanguageStore; + children: React.ReactNode; +}; + +const neverChanges = () => () => undefined; + +export function LanguageGate({ i18n, locales, store, children }: LanguageGateProps) { + const language = useSyncExternalStore( + neverChanges, + () => resolveLanguage(store.read(), locales), + () => DEFAULT_LANGUAGE, + ); + const [settled, setSettled] = useState(DEFAULT_LANGUAGE); + + useEffect(() => { + const load = locales.get(language); + if (!load) return; + load() + .then((messages) => { + i18n.addResourceBundle(language, "translation", messages); + return i18n.changeLanguage(language); + }) + .then(() => { + document.documentElement.lang = language; + }) + .catch((error: unknown) => console.warn(`Couldn't load the "${language}" translations`, error)) + .finally(() => setSettled(language)); + }, [i18n, language, locales]); + + return language === settled ? children : ; +} export default function I18nProvider({ children }: { children: React.ReactNode }) { - return {children}; + return ( + + + {children} + + + ); } diff --git a/ui/litellm-dashboard/src/i18n/languages.test.ts b/ui/litellm-dashboard/src/i18n/languages.test.ts new file mode 100644 index 00000000000..5a462d7a52d --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/languages.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; + +import { englishName, Locales, nativeName, resolveLanguage, supportedLanguages } from "./languages"; + +const locales: Locales = new Map([["ja", async () => ({})]]); + +describe("resolveLanguage", () => { + it("uses the stored language when it has translations", () => { + expect(resolveLanguage("ja", locales)).toBe("ja"); + }); + + it.each([null, "", "en", "fr", "JA", "ja-JP", "__proto__"])("shows English when the stored value is %j", (stored) => { + expect(resolveLanguage(stored, locales)).toBe("en"); + }); + + it("shows English when no language besides English has translations", () => { + expect(resolveLanguage("ja", new Map())).toBe("en"); + }); +}); + +describe("supportedLanguages", () => { + it("puts English first, then every language with translations", () => { + expect(supportedLanguages(locales)).toEqual(["en", "ja"]); + expect(supportedLanguages(new Map())).toEqual(["en"]); + }); +}); + +describe("language names", () => { + it("names a language in itself, starting with a capital letter", () => { + const spanish = nativeName("es"); + + expect(spanish.toLocaleLowerCase("es")).toBe( + new Intl.DisplayNames(["es"], { type: "language" }).of("es")?.toLocaleLowerCase("es"), + ); + expect(spanish.charAt(0)).toBe(spanish.charAt(0).toLocaleUpperCase("es")); + }); + + it("names a language in English for the secondary label", () => { + expect(englishName("ja")).toBe(new Intl.DisplayNames(["en"], { type: "language" }).of("ja")); + expect(englishName("ja")).not.toBe(nativeName("ja")); + }); +}); diff --git a/ui/litellm-dashboard/src/i18n/languages.ts b/ui/litellm-dashboard/src/i18n/languages.ts new file mode 100644 index 00000000000..616523a7e54 --- /dev/null +++ b/ui/litellm-dashboard/src/i18n/languages.ts @@ -0,0 +1,30 @@ +import { getLocalStorageItem, setLocalStorageItem } from "@/utils/localStorageUtils"; + +export type Messages = { readonly [key: string]: string | Messages }; +export type Locales = ReadonlyMap Promise>; +export type LanguageStore = { readonly read: () => string | null; readonly write: (language: string) => void }; + +export const DEFAULT_LANGUAGE = "en"; +export const LANGUAGE_STORAGE_KEY = "litellm_language"; + +export const LOCALES: Locales = new Map(); + +export const browserLanguageStore: LanguageStore = { + read: () => getLocalStorageItem(LANGUAGE_STORAGE_KEY), + write: (language) => setLocalStorageItem(LANGUAGE_STORAGE_KEY, language), +}; + +export const resolveLanguage = (stored: string | null, locales: Locales): string => + stored !== null && locales.has(stored) ? stored : DEFAULT_LANGUAGE; + +export const supportedLanguages = (locales: Locales): readonly string[] => [DEFAULT_LANGUAGE, ...locales.keys()]; + +const displayName = (language: string, displayLanguage: string): string => + new Intl.DisplayNames([displayLanguage], { type: "language" }).of(language) ?? language; + +export const nativeName = (language: string): string => { + const name = displayName(language, language); + return name.charAt(0).toLocaleUpperCase(language) + name.slice(1); +}; + +export const englishName = (language: string): string => displayName(language, DEFAULT_LANGUAGE); diff --git a/ui/litellm-dashboard/src/i18n/locales/en.json b/ui/litellm-dashboard/src/i18n/locales/en.json index f8886610e01..c72b9b91ca7 100644 --- a/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/ui/litellm-dashboard/src/i18n/locales/en.json @@ -8085,6 +8085,10 @@ "copyUserId": "Copy user ID", "hideAria": "Hide ", "toggleHideAria": "Toggle hide {{name}}", + "language": { + "label": "Language", + "fallbackNote": "Missing translations show in English. Help translate" + }, "changePassword": "Change Password", "logout": "Logout" }, From 8ac9da84f7ba6b79fbf101d5ebe0ccadc91e529b Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Thu, 5 Nov 2026 17:58:17 -0800 Subject: [PATCH 32/34] fix(router): drop supports_web_search false deployments for Anthropic web_search_* and Gemini googleSearch tools (#50275) * fix(router): drop supports_web_search false deployments for Anthropic web_search_* and Gemini googleSearch tools * test(integration): build the web search filter proxy config without mutating the shared config --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../routing_common_utils.py | 13 +- ...eb_search_deployment_filter_owned_proxy.py | 146 ++++++++++++++++++ .../test_router_utils_common_utils.py | 29 ++++ 3 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py diff --git a/litellm/litellm_core_utils/routing_common_utils.py b/litellm/litellm_core_utils/routing_common_utils.py index 62bdc1c10a7..d13ff4059aa 100644 --- a/litellm/litellm_core_utils/routing_common_utils.py +++ b/litellm/litellm_core_utils/routing_common_utils.py @@ -409,7 +409,7 @@ def deployments_the_caller_may_use( return () -_WEB_SEARCH_TOOL_TYPES: Final = frozenset({"web_search", "web_search_preview"}) +_WEB_SEARCH_TOOL_KEYS: Final = frozenset({"googleSearch", "google_search"}) def _deployment_supports_web_search(deployment: dict) -> bool: @@ -433,11 +433,18 @@ def _deployment_supports_web_search(deployment: dict) -> bool: def is_web_search_request(request_kwargs: Mapping[str, object]) -> bool: + """OpenAI ``web_search`` / ``web_search_preview``, Anthropic ``web_search_``, Gemini + ``googleSearch`` / ``google_search``.""" tools: Final[object] = request_kwargs.get("tools") return isinstance(tools, (list, tuple)) and any( isinstance(tool, dict) - and isinstance(tool_type := tool.get("type"), str) - and tool_type in _WEB_SEARCH_TOOL_TYPES + and ( + any(key in tool for key in _WEB_SEARCH_TOOL_KEYS) + or ( + isinstance(tool_type := tool.get("type"), str) + and (tool_type == "web_search" or tool_type.startswith("web_search_")) + ) + ) for tool in tools ) diff --git a/tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py b/tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py new file mode 100644 index 00000000000..2292ddd8611 --- /dev/null +++ b/tests/integration/routing/test_web_search_deployment_filter_owned_proxy.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import json +import uuid +from collections.abc import Iterator, Mapping +from pathlib import Path +from typing import Final + +import pytest +import yaml +from pydantic import JsonValue + +from tests.integration._support.anthropic_sse import message_json +from tests.integration._support.client import Gateway, gateway_from_environment +from tests.integration._support.process import owned_proxy +from tests.integration._support.provider import PROVIDER_URL, SharedProvider +from tests.integration._support.wire import Reply + +_ANTHROPIC_GROUP: Final = "web-search-anthropic" +_GEMINI_GROUP: Final = "web-search-gemini" +_REQUESTS: Final = 10 +_PROMPT: Final = "Reply with OK." + + +def _gemini_json() -> bytes: + return json.dumps( + { + "candidates": [ + {"content": {"parts": [{"text": "OK"}], "role": "model"}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 1, "totalTokenCount": 6}, + "modelVersion": "gemini-3.8-flash", + } + ).encode() + + +def _supported(group: str) -> str: + return f"{group}-supported" + + +def _unsupported(group: str) -> str: + return f"{group}-unsupported" + + +def _group(group: str, model: str, api_base: str) -> tuple[dict[str, JsonValue], ...]: + # The unsupported deployment is weighted so a router that keeps it in rotation picks it + # nearly every time, rather than half the time by chance + return ( + { + "model_name": group, + "litellm_params": {"model": model, "api_key": "sk-fixture", "api_base": api_base, "weight": 1}, + "model_info": {"id": _supported(group)}, + }, + { + "model_name": group, + "litellm_params": {"model": model, "api_key": "sk-fixture", "api_base": api_base, "weight": 9}, + "model_info": {"id": _unsupported(group), "supports_web_search": False}, + }, + ) + + +def _tools(tool: Mapping[str, object]) -> dict[str, object]: + return {"tools": [dict(tool)]} + + +def _config(directory: Path) -> Path: + shared: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + configuration: Final = { + **shared, + "model_list": [ + *shared["model_list"], + *_group(_ANTHROPIC_GROUP, "anthropic/claude-haiku-5-5", PROVIDER_URL), + *_group(_GEMINI_GROUP, "gemini/gemini-3.8-flash", PROVIDER_URL), + ], + "router_settings": {**shared.get("router_settings", {}), "routing_strategy": "simple-shuffle"}, + } + path: Final = directory / "config-web-search-filter.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +@pytest.fixture(scope="module") +def web_search_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("web-search-filter") + with ( + gateway_from_environment() as shared, + owned_proxy(shared, directory, {}, config=_config(directory)) as owned, + ): + yield owned + + +@pytest.mark.parametrize( + ("path", "group", "body", "reply"), + [ + pytest.param( + "/v1/chat/completions", + _ANTHROPIC_GROUP, + _tools({"type": "web_search_20250305", "name": "web_search", "max_uses": 1}), + message_json("msg_web_search", "claude-haiku-5-5", "OK"), + id="anthropic-web_search_20250305-chat", + ), + pytest.param( + "/v1/messages", + _ANTHROPIC_GROUP, + {"max_tokens": 16, **_tools({"type": "web_search_20260301", "name": "web_search"})}, + message_json("msg_web_search", "claude-haiku-5-5", "OK"), + id="anthropic-web_search_newer-version-messages", + ), + pytest.param( + "/v1/chat/completions", + _GEMINI_GROUP, + _tools({"googleSearch": {}}), + _gemini_json(), + id="gemini-googleSearch-chat", + ), + pytest.param( + "/v1/chat/completions", + _GEMINI_GROUP, + _tools({"google_search": {}}), + _gemini_json(), + id="gemini-google_search-chat", + ), + ], +) +def test_a_web_search_request_never_reaches_a_deployment_declared_without_web_search( + web_search_proxy: Gateway, + provider: SharedProvider, + path: str, + group: str, + body: Mapping[str, JsonValue], + reply: bytes, +) -> None: + provider.expect(*(Reply(body=reply) for _ in range(_REQUESTS))) + responses: Final = tuple( + web_search_proxy.request( + "POST", + path, + {"model": group, "messages": [{"role": "user", "content": f"{_PROMPT} {uuid.uuid4().hex}"}], **body}, + ) + for _ in range(_REQUESTS) + ) + received: Final = provider.received() + assert all(response.status_code == 200 for response in responses), [response.text for response in responses] + served: Final = tuple(response.headers["x-litellm-model-id"] for response in responses) + assert served == (_supported(group),) * _REQUESTS, served + assert len(received) == _REQUESTS diff --git a/tests/unit/router_utils/test_router_utils_common_utils.py b/tests/unit/router_utils/test_router_utils_common_utils.py index 0de10a64f2e..42b21692a8f 100644 --- a/tests/unit/router_utils/test_router_utils_common_utils.py +++ b/tests/unit/router_utils/test_router_utils_common_utils.py @@ -285,6 +285,35 @@ class TestFilterWebSearchDeployments: ) assert result == sample_deployments + @pytest.mark.parametrize( + "tool", + [ + {"type": "web_search_20250305", "name": "web_search", "max_uses": 1}, + {"type": "web_search_20260301", "name": "web_search"}, + {"googleSearch": {}}, + {"google_search": {}}, + ], + ids=["anthropic_versioned", "anthropic_newer_version", "gemini_camel_case", "gemini_snake_case"], + ) + def test_a_provider_native_web_search_tool_filters_unsupported( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + result: Final = filter_web_search_deployments(sample_deployments, {"tools": [{"type": "function"}, tool]}) + assert [d["model_info"]["id"] for d in result] == ["deployment-1", "deployment-2"] + + @pytest.mark.parametrize( + "tool", + [ + {"type": "function", "function": {"name": "web_search"}}, + {"codeExecution": {}}, + ], + ids=["function_named_web_search", "gemini_code_execution"], + ) + def test_a_tool_that_is_no_web_search_keeps_every_deployment( + self, sample_deployments: list[dict[str, object]], tool: dict[str, object] + ) -> None: + assert filter_web_search_deployments(sample_deployments, {"tools": [tool]}) == sample_deployments + def test_web_search_preview_filters_unsupported(self, sample_deployments): """web_search_preview type should also trigger filtering""" request_kwargs = {"tools": [{"type": "web_search_preview"}]} From 5085022580dede5339f8fe957d0f91ba7dc0daa0 Mon Sep 17 00:00:00 2001 From: qtim0 <154377292+qtim0@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:07:15 -0500 Subject: [PATCH 33/34] fix(audio-transcription): keep transcripts out of transcription error messages (#50137) * fix(gemini): omit raw response body from audio transcription error messages * fix(gemini): raise incomplete transcription interactions as non-retryable 400 and drop response bodies from Vertex transcription errors * fix(audio_transcription): keep response bodies out of transcription errors Keep the upstream status on a Gemini interaction that did not complete, so the router still retries it as it does on main. Gemini stops the same long clip on some tries and not on others, so a retry can recover it. Stop quoting the reply in the Deepgram, ElevenLabs, Watsonx, Mistral, OVHcloud and Scaleway parse errors, and hide pydantic input values on the Gemini and Vertex response models. * fix(audio-transcription): keep reply bodies out of watsonx, whisper, xai and validation errors Watsonx and whisper no longer print the parsed reply after Got: when it has no text field. TranscriptionResponse and the xAI response model hide input in validation errors, so a malformed field cannot quote transcript fragments. Deepgram and ElevenLabs keep the original error text, which is now safe. * fix(audio-transcription): keep transcripts out of the shared converter's parse errors * test(audio-transcription): compare wrapped lookup errors instead of pinning upstream wording --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../convert_dict_to_response.py | 5 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 4 +- .../audio_transcription/transformation.py | 2 +- .../transcriptions/whisper_transformation.py | 5 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../gemini_transcribe_transformation.py | 2 +- .../audio_transcription/transformation.py | 2 +- .../audio_transcription/transformation.py | 7 +- .../xai/audio_transcription/transformation.py | 2 +- .../types/llms/gemini_audio_transcription.py | 2 +- .../llms/vertex_ai_gemini_transcription.py | 2 +- .../types/llms/vertex_ai_speech_to_text.py | 2 + litellm/types/utils.py | 2 + .../test_convert_dict_to_response.py | 18 +++++ ...gram_audio_transcription_transformation.py | 28 ++++--- .../test_transformation.py | 35 ++++++--- ...mini_audio_transcription_transformation.py | 77 +++++++++++++++++++ ...tral_audio_transcription_transformation.py | 22 ++++++ .../test_whisper_transformation.py | 9 +++ ...loud_audio_transcription_transformation.py | 10 +++ ...eway_audio_transcription_transformation.py | 11 +++ ...x_ai_audio_transcription_transformation.py | 14 +++- ...tex_ai_gemini_transcribe_transformation.py | 15 ++++ ...sonx_audio_transcription_transformation.py | 26 +++++-- ..._xai_audio_transcription_transformation.py | 19 +++++ 28 files changed, 276 insertions(+), 53 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 9fb4ad27d75..0fed2cb2fce 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -1072,9 +1072,10 @@ def convert_to_model_response_object( if isinstance(e, APIError): raise + hide_reply: Final = response_type == "audio_transcription" received_args: Final = dict( - response_object=response_object, - model_response_object=model_response_object, + response_object="" if hide_reply else response_object, + model_response_object="" if hide_reply else model_response_object, response_type=response_type, stream=stream, start_time=start_time, diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index 1c8a056116f..2c8b22459bd 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -158,7 +158,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming Deepgram response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming Deepgram response: {e}") from None def _reconstruct_diarized_transcript(self, words: list) -> str: """ diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 3dc42f35d3c..b09aa97f6fe 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -160,7 +160,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): return response except Exception as e: - raise ValueError(f"Error transforming ElevenLabs response: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming ElevenLabs response: {e}") from None def get_complete_url( self, diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 3457b22faa4..ca4953445ee 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -138,13 +138,13 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): except ValueError: raise GeminiError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Gemini Interactions API: {raw_response.text}", + message="Received non-JSON response from Gemini Interactions API", ) parsed: Final = GeminiTranscriptionInteractionResponse.model_validate(response_json) if parsed.status != "completed": raise GeminiError( status_code=raw_response.status_code, - message=f"Gemini transcription interaction did not complete (status={parsed.status}): {raw_response.text}", + message=f"Gemini transcription interaction did not complete (status={parsed.status})", ) text_contents: Final = tuple( content diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 3eb9e122265..34200effb18 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -133,7 +133,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise MistralAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Mistral audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index eb4f1b6c9ce..947d6175122 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -141,7 +141,4 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if any(key in raw_response_json for key in TranscriptionResponse.model_fields): return TranscriptionResponse(**raw_response_json) else: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - raw_response_json, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index dd61bfb0c13..50bed1fef78 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -144,7 +144,7 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise OVHCloudException( - message=raw_response.text, + message="Received non-JSON response from OVHCloud audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/scaleway/audio_transcription/transformation.py b/litellm/llms/scaleway/audio_transcription/transformation.py index 8913cc362b3..2cc3ac17073 100644 --- a/litellm/llms/scaleway/audio_transcription/transformation.py +++ b/litellm/llms/scaleway/audio_transcription/transformation.py @@ -134,7 +134,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig): response_json: Final = raw_response.json() except Exception: raise ScalewayAudioTranscriptionException( - message=raw_response.text, + message="Received non-JSON response from Scaleway audio transcription", status_code=raw_response.status_code, headers=raw_response.headers, ) diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index 26de755d5ca..09a09ef96dc 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -183,7 +183,7 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Vertex AI Gemini transcription: {raw_response.text}", + message="Received non-JSON response from Vertex AI Gemini transcription", ) parsed: Final = VertexGeminiTranscriptionResponse.model_validate(response_json) texts: Final = tuple( diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py index 4fdf93a6307..017d7ff7b2f 100644 --- a/litellm/llms/vertex_ai/audio_transcription/transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -172,7 +172,7 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase) except ValueError: raise VertexAIError( status_code=raw_response.status_code, - message=f"Received non-JSON response from Google Speech-to-Text: {raw_response.text}", + message="Received non-JSON response from Google Speech-to-Text", ) parsed: Final = VertexSpeechToTextRecognizeResponse.model_validate(response_json) transcripts: Final = tuple( diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 35fda05c028..b160ee65802 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -175,7 +175,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran try: raw_response_json: Final = _DECODED_JSON.validate_python(raw_response.json()) except Exception as e: - raise ValueError(f"Error transforming response to json: {e}\nResponse: {raw_response.text}") + raise ValueError(f"Error transforming response to json: {e}") # Extract only valid fields for TranscriptionResponse.__init__() # TranscriptionResponse only accepts 'text' and 'usage' in __init__() @@ -191,10 +191,7 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran response_kwargs["usage"] = usage if not response_kwargs: - raise ValueError( - "Invalid response format. Received response does not match the expected format. Got: ", - response_object, - ) + raise ValueError("Invalid response format. Received response does not match the expected format.") response: Final = TranscriptionResponse(**response_kwargs) diff --git a/litellm/llms/xai/audio_transcription/transformation.py b/litellm/llms/xai/audio_transcription/transformation.py index e29c3d9334a..c212dce26fe 100644 --- a/litellm/llms/xai/audio_transcription/transformation.py +++ b/litellm/llms/xai/audio_transcription/transformation.py @@ -39,7 +39,7 @@ class _XAISttWord(LiteLLMBaseModel): class _XAISttResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="allow") + model_config = ConfigDict(extra="allow", hide_input_in_errors=True) text: str = "" language: str = "unknown" duration: float | None = None diff --git a/litellm/types/llms/gemini_audio_transcription.py b/litellm/types/llms/gemini_audio_transcription.py index c1cf3452945..59dd15aa13e 100644 --- a/litellm/types/llms/gemini_audio_transcription.py +++ b/litellm/types/llms/gemini_audio_transcription.py @@ -75,7 +75,7 @@ class GeminiTranscriptionUsage(LiteLLMBaseModel): class GeminiTranscriptionInteractionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) id: str | None = None status: str | None = None diff --git a/litellm/types/llms/vertex_ai_gemini_transcription.py b/litellm/types/llms/vertex_ai_gemini_transcription.py index e78aafaca9f..d7bc136bcd1 100644 --- a/litellm/types/llms/vertex_ai_gemini_transcription.py +++ b/litellm/types/llms/vertex_ai_gemini_transcription.py @@ -68,7 +68,7 @@ class VertexGeminiTranscriptionUsageMetadata(LiteLLMBaseModel): class VertexGeminiTranscriptionResponse(LiteLLMBaseModel): - model_config = ConfigDict(extra="ignore") + model_config = ConfigDict(extra="ignore", hide_input_in_errors=True) candidates: tuple[VertexGeminiTranscriptionCandidate, ...] = () usageMetadata: VertexGeminiTranscriptionUsageMetadata | None = None diff --git a/litellm/types/llms/vertex_ai_speech_to_text.py b/litellm/types/llms/vertex_ai_speech_to_text.py index f3d343d13ce..3dea7610150 100644 --- a/litellm/types/llms/vertex_ai_speech_to_text.py +++ b/litellm/types/llms/vertex_ai_speech_to_text.py @@ -40,6 +40,8 @@ class VertexSpeechToTextResponseMetadata(LiteLLMBaseModel): class VertexSpeechToTextRecognizeResponse(LiteLLMBaseModel): + model_config = ConfigDict(hide_input_in_errors=True) + results: list[VertexSpeechToTextResult] = Field(default=[]) metadata: VertexSpeechToTextResponseMetadata | None = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ca434154f68..7c137b72c3e 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2989,6 +2989,8 @@ _DETECTED_LANGUAGES: Final = TypeAdapter(tuple[TranscriptionDetectedLanguage, .. class TranscriptionResponse(OpenAIObject): + model_config = ConfigDict(hide_input_in_errors=True) + text: str | None = None usage: TranscriptionUsageDurationObject | TranscriptionUsageTokensObject | None = None languages: Sequence[TranscriptionDetectedLanguage] | None = None diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py index 3e197740e98..c41073e087f 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_convert_dict_to_response.py @@ -2814,6 +2814,24 @@ class TestConvertToModelResponseObjectAudioTranscription: assert result.usage.output_tokens == 5 assert result.usage.input_token_details.audio_tokens == 4 + def test_parse_error_keeps_the_transcript_out_of_the_exception(self): + import traceback + + from litellm.types.utils import TranscriptionResponse + + transcript: Final = "CANARYTRANSCRIPT customer account ending 2750" + + with pytest.raises(Exception, match="Invalid response object") as exc_info: + convert_to_model_response_object( + response_object={"text": transcript, "usage": {"type": "duration", "seconds": "not-a-number"}}, + model_response_object=TranscriptionResponse(), + response_type="audio_transcription", + ) + + rendered: Final = "".join(traceback.format_exception(exc_info.value)) + assert "received_args=" in rendered + assert "CANARYTRANSCRIPT" not in rendered + class TestConvertToModelResponseObjectRerank: def test_basic_rerank(self): from litellm.types.utils import RerankResponse diff --git a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py index 0951ba6ba92..d298142fe4b 100644 --- a/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py +++ b/tests/unit/llms/deepgram/audio_transcription/test_deepgram_audio_transcription_transformation.py @@ -1,11 +1,13 @@ import io import os import pathlib +import traceback from typing import Final from unittest.mock import MagicMock import httpx import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import AudioTranscriptionRequestData from litellm.llms.deepgram.audio_transcription.transformation import ( @@ -588,31 +590,33 @@ def test_transform_response_maps_words_to_openai_word_timestamps(words: object, [{"word": "hello", "start": 0.0}], ], ) -def test_transform_response_wraps_malformed_words_with_the_raw_body(words: object): +def test_transform_response_keeps_the_transcript_out_of_malformed_words_errors(words: object): raw_response = httpx.Response(200, json=_deepgram_payload({"transcript": "hello", "words": words}, {})) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize( - ("payload", "error"), + ("payload", "failing_lookup_error"), [ - ({"results": None}, "'NoneType' object is not subscriptable"), - (_deepgram_payload({"transcript": "hello", "words": 5}, {}), "object of type 'int' has no len()"), - ( - {**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, - "'NoneType' object is not subscriptable", - ), + ({"results": None}, TypeError), + (_deepgram_payload({"transcript": "hello", "words": 5}, {}), TypeError), + ({**_deepgram_payload({"transcript": "hello"}, {}), "metadata": None}, TypeError), + (_deepgram_payload({"transcript": ["hello"]}, {}), ValidationError), ], ) -def test_transform_response_wraps_the_error_of_the_failing_lookup_in_a_malformed_body(payload: object, error: str): +def test_transform_response_wraps_the_error_of_the_failing_lookup_without_the_body( + payload: object, failing_lookup_error: type[Exception] +): raw_response: Final = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming Deepgram response: ") as exc_info: DeepgramAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) - assert exc_info.type is ValueError - assert str(exc_info.value) == f"Error transforming Deepgram response: {error}\nResponse: {raw_response.text}" + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, failing_lookup_error) + assert str(exc_info.value) == f"Error transforming Deepgram response: {failing_lookup}" + assert "hello" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py index f55644eccae..2f5754b3765 100644 --- a/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py +++ b/tests/unit/llms/elevenlabs/audio_transcription/test_transformation.py @@ -1,9 +1,11 @@ +import traceback from collections.abc import Iterator from typing import Final import httpx import pytest import respx +from pydantic import ValidationError import litellm from litellm.caching.llm_caching_handler import LLMClientCache @@ -109,24 +111,35 @@ def test_transform_audio_transcription_response_without_words_key_has_no_word_li @pytest.mark.parametrize( "payload", [ - ["not", "an", "object"], - "plain text", - 7, - {"text": ["not", "text"]}, - {"text": "t", "words": None}, - {"text": "t", "words": 7}, - {"text": "t", "words": "not a list"}, - {"text": "t", "words": ["not an object"]}, - {"text": "t", "words": [{"type": "word", "text": "ok"}, None]}, + ["sensitive", "an", "object"], + "sensitive text", + {"text": ["sensitive", "text"]}, + {"text": "sensitive", "words": None}, + {"text": "sensitive", "words": 7}, + {"text": "sensitive", "words": "not a list"}, + {"text": "sensitive", "words": ["not an object"]}, + {"text": "sensitive", "words": [{"type": "word", "text": "ok"}, None]}, ], ) -def test_transform_audio_transcription_response_wraps_malformed_payloads_with_the_raw_body(payload: object): +def test_transform_audio_transcription_response_keeps_the_body_out_of_malformed_payload_errors(payload: object): raw_response = httpx.Response(200, json=payload) with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) - assert str(exc_info.value).endswith(f"\nResponse: {raw_response.text}") + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) + + +def test_transform_audio_transcription_response_names_the_failing_lookup_of_a_malformed_payload(): + raw_response = httpx.Response(200, json={"text": "sensitive", "words": [None]}) + + with pytest.raises(ValueError, match="Error transforming ElevenLabs response: ") as exc_info: + ElevenLabsAudioTranscriptionConfig().transform_audio_transcription_response(raw_response=raw_response) + + failing_lookup: Final = exc_info.value.__context__ + assert isinstance(failing_lookup, ValidationError) + assert str(exc_info.value) == f"Error transforming ElevenLabs response: {failing_lookup}" + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) class TestElevenLabsAudioTranscription: diff --git a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py index 08084c8fac0..a9ef468525f 100644 --- a/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py +++ b/tests/unit/llms/gemini/audio_transcription/test_gemini_audio_transcription_transformation.py @@ -1,8 +1,10 @@ import base64 import json +import traceback import httpx import pytest +import respx import litellm from litellm.llms.gemini.audio_transcription.transformation import ( @@ -241,6 +243,21 @@ class TestTransformResponse: with pytest.raises(GeminiError, match="non-JSON"): config.transform_audio_transcription_response(raw) + def test_non_completed_status_error_omits_response_body(self, config): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = "incomplete" + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(make_response(payload)) + assert "status=incomplete" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + def test_non_json_error_omits_response_body(self, config): + raw = httpx.Response(200, text="sensitive body", request=httpx.Request("POST", "https://example.test")) + with pytest.raises(GeminiError) as excinfo: + config.transform_audio_transcription_response(raw) + assert "sensitive body" not in str(excinfo.value) + def test_word_without_offsets_survives(self, config): payload = json.loads(json.dumps(COMPLETED_RESPONSE)) payload["steps"][0]["content"][0]["annotations"] = [{"type": "word_info", "text": "Hello"}] @@ -249,6 +266,66 @@ class TestTransformResponse: assert response.get("duration") is None + +class TestNonCompletedInteractionThroughRouter: + @pytest.fixture + def router(self, monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return litellm.Router( + model_list=[ + { + "model_name": "transcribe", + "litellm_params": {"model": "gemini/gemini-3.5-transcribe", "api_key": "test-gemini-key"}, + } + ], + num_retries=2, + ) + + def _interaction(self, status): + payload = json.loads(json.dumps(COMPLETED_RESPONSE)) + payload["status"] = status + payload["steps"][0]["content"][0]["text"] = "sensitive transcript text" + return payload + + @pytest.mark.asyncio + @respx.mock + async def test_retry_recovers_an_incomplete_interaction(self, router): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + side_effect=[ + httpx.Response(200, json=self._interaction("incomplete")), + httpx.Response(200, json=COMPLETED_RESPONSE), + ] + ) + response = await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 2 + assert response.text == COMPLETED_RESPONSE["steps"][0]["content"][0]["text"] + + @pytest.mark.asyncio + @respx.mock + @pytest.mark.parametrize("status", ["incomplete", "failed", "cancelled", "in_progress", None]) + async def test_non_completed_interaction_retries_then_fails_without_transcript(self, router, status): + route = respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=self._interaction(status)) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert route.call_count == 3 + assert f"status={status}" in str(excinfo.value) + assert "sensitive transcript text" not in str(excinfo.value) + + @pytest.mark.asyncio + @respx.mock + async def test_response_field_of_unexpected_type_keeps_transcript_out_of_the_error(self, router): + payload = self._interaction("incomplete") + payload["steps"][0]["content"] = "sensitive transcript text" + respx.post("https://generativelanguage.googleapis.com/v1beta/interactions").mock( + return_value=httpx.Response(200, json=payload) + ) + with pytest.raises(litellm.APIConnectionError) as excinfo: + await router.atranscription(model="transcribe", file=("call.wav", AUDIO_BYTES, "audio/wav")) + assert "sensitive transcript text" not in "".join(traceback.format_exception(excinfo.value)) + + class TestSubtitleSynthesisThroughHandler: def _transform(self, config, response_format): from unittest.mock import Mock diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index b475cf7e3ff..494b1fe34ad 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -3,12 +3,15 @@ from unittest.mock import MagicMock import httpx import litellm +import pytest +from pydantic import ValidationError from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) from litellm.llms.mistral.audio_transcription.transformation import ( MistralAudioTranscriptionConfig, + MistralAudioTranscriptionException, ) from litellm.types.utils import TranscriptionResponse from litellm.utils import ProviderConfigManager @@ -191,3 +194,22 @@ def test_mistral_audio_transcription_response_transform_empty(): assert isinstance(response, TranscriptionResponse) assert response.text == "" + + +def test_mistral_audio_transcription_response_transform_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(MistralAudioTranscriptionException, match="non-JSON") as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + +@pytest.mark.parametrize("text", [["sensitive"], {"value": "sensitive"}]) +def test_mistral_audio_transcription_response_transform_keeps_a_malformed_text_field_out_of_the_error(text: object): + raw_response = httpx.Response(200, json={"text": text, "segments": [{"text": "sensitive"}]}) + + with pytest.raises(ValidationError) as exc_info: + MistralAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py index 2dc24b1b313..339d62ef7ea 100644 --- a/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py +++ b/tests/unit/llms/openai/transcriptions/test_whisper_transformation.py @@ -5,8 +5,10 @@ and transform_audio_transcription_response. import io import json +import traceback from unittest.mock import MagicMock +import httpx import pytest from litellm.llms.openai.transcriptions.whisper_transformation import ( @@ -104,3 +106,10 @@ class TestWhisperTransformResponse: is_json=False, ) ) + + def test_json_object_without_transcription_fields_keeps_the_body_out_of_the_error(self): + config = OpenAIWhisperAudioTranscriptionConfig() + raw_response = httpx.Response(200, json={"segments": [{"text": "sensitive"}]}) + with pytest.raises(ValueError, match="Invalid response format") as exc_info: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) diff --git a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py index 422658932e6..ed8ec88e209 100644 --- a/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py +++ b/tests/unit/llms/ovhcloud/test_ovhcloud_audio_transcription_transformation.py @@ -3,6 +3,7 @@ import pytest from pydantic import ValidationError from litellm.llms.ovhcloud.audio_transcription.transformation import OVHCloudAudioTranscriptionConfig +from litellm.llms.ovhcloud.utils import OVHCloudException from litellm.types.utils import TranscriptionResponse @@ -98,3 +99,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies(payloa _transform(payload) assert "spoken secret" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error(): + raw_response = httpx.Response(200, text='{"text": "spoken secret') + + with pytest.raises(OVHCloudException, match="non-JSON") as exc_info: + OVHCloudAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "spoken secret" not in str(exc_info.value) diff --git a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py index f313f8bd873..16e60a15539 100644 --- a/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py +++ b/tests/unit/llms/scaleway/test_scaleway_audio_transcription_transformation.py @@ -204,6 +204,17 @@ def test_scaleway_transform_response_raises_typed_exception_on_non_json(): ) +def test_scaleway_transform_response_keeps_non_json_body_out_of_the_error(): + raw_response = httpx.Response( + 200, text='{"text": "sensitive', headers={"content-type": "application/json"} + ) + + with pytest.raises(ScalewayAudioTranscriptionException, match="non-JSON") as exc_info: + ScalewayAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) + + def test_scaleway_transform_response_returns_plain_text_for_non_json_content_type(): """When Scaleway responds with text/srt/vtt (response_format="text" etc.), the content-type is not application/json — return the body as plain text diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py index 5898d933941..02b7c0ae71a 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_audio_transcription_transformation.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse import httpx import pytest - +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.transformation import ( @@ -208,6 +208,18 @@ class TestTransformResponse: response = config.transform_audio_transcription_response(raw_response) assert response.text == "" + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"results": [{"alternatives": [{"transcript": "sensitive') + with pytest.raises(VertexAIError, match="non-JSON") as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, json={"results": "sensitive transcript"}) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + def test_fractional_billed_duration(self, config): raw_response = httpx.Response( status_code=200, diff --git a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py index 82ea034f91b..532e013c3d7 100644 --- a/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py +++ b/tests/unit/llms/vertex_ai/audio_transcription/test_vertex_ai_gemini_transcribe_transformation.py @@ -4,6 +4,7 @@ import os import httpx import pytest +from pydantic import ValidationError import litellm from litellm.llms.vertex_ai.audio_transcription.gemini_transcribe_transformation import ( @@ -255,6 +256,20 @@ class TestTransformResponse: with pytest.raises(VertexAIError, match="non-JSON"): config.transform_audio_transcription_response(raw_response) + def test_non_json_error_omits_response_body(self, config): + raw_response = httpx.Response(status_code=200, text='{"candidates": [{"content": {"parts": [{"text": "sensitive') + with pytest.raises(VertexAIError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + + def test_field_of_unexpected_type_omits_response_body(self, config): + raw_response = httpx.Response( + status_code=200, json={"candidates": [{"content": {"parts": "sensitive transcript"}}]} + ) + with pytest.raises(ValidationError) as excinfo: + config.transform_audio_transcription_response(raw_response) + assert "sensitive" not in str(excinfo.value) + class TestValidateEnvironment: def test_sets_oauth_headers(self): diff --git a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py index b6e5007c456..c9f6f566ad6 100644 --- a/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py +++ b/tests/unit/llms/watsonx/audio_transcription/test_watsonx_audio_transcription_transformation.py @@ -4,6 +4,7 @@ Tests for IBM WatsonX Audio Transcription. Validates the WatsonX transcription response transformation. """ +import traceback from unittest.mock import MagicMock import httpx @@ -120,17 +121,21 @@ def test_transform_audio_transcription_response_copies_every_field_except_model( assert {key: response[key] for key in expected_extras} == expected_extras -@pytest.mark.parametrize("payload", [{}, {"model": "whisper"}, {"duration": 2.0}, {"text": None, "usage": None}]) -def test_transform_audio_transcription_response_without_text_or_usage_reports_the_body( +@pytest.mark.parametrize( + "payload", + [ + {"model": "sensitive"}, + {"duration": 2.0, "segments": [{"text": "sensitive"}]}, + {"text": None, "usage": None, "note": "sensitive"}, + ], +) +def test_transform_audio_transcription_response_without_text_or_usage_keeps_the_body_out_of_the_error( payload: dict[str, object], ) -> None: with pytest.raises(ValueError, match="Invalid response format") as exc_info: _transform_transcription_response(payload) - assert exc_info.value.args == ( - "Invalid response format. Received response does not match the expected format. Got: ", - payload, - ) + assert "sensitive" not in "".join(traceback.format_exception(exc_info.value)) @pytest.mark.parametrize("payload", [["not", "an", "object"], "plain text", 7, True]) @@ -141,3 +146,12 @@ def test_transform_audio_transcription_response_rejects_non_object_bodies_withou _transform_transcription_response(payload) assert "input_value" not in str(exc_info.value) + + +def test_transform_audio_transcription_response_keeps_a_non_json_body_out_of_the_error() -> None: + raw_response = httpx.Response(200, text='{"text": "sensitive') + + with pytest.raises(ValueError, match="Error transforming response to json") as exc_info: + IBMWatsonXAudioTranscriptionConfig().transform_audio_transcription_response(raw_response) + + assert "sensitive" not in str(exc_info.value) diff --git a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py index e2e3fc3d4dc..3505116819b 100644 --- a/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py +++ b/tests/unit/llms/xai/test_xai_audio_transcription_transformation.py @@ -1,3 +1,5 @@ +import traceback + import httpx import pytest @@ -146,6 +148,23 @@ def test_transform_response_raises_on_error_status(): assert "Incorrect API key provided" in exc.value.message +@pytest.mark.parametrize( + "content", + [ + b'{"text": ["sensitive"]}', + b'{"text": "ok", "words": [{"text": ["sensitive"]}]}', + b'{"text": "sensitive', + ], +) +def test_transform_response_keeps_a_malformed_body_out_of_the_error(content): + raw = httpx.Response(200, content=content, request=httpx.Request("POST", "https://api.x.ai/v1/stt")) + + with pytest.raises(XAIAudioTranscriptionError, match="Error parsing xAI response") as exc: + CONFIG.transform_audio_transcription_response(raw_response=raw) + + assert "sensitive" not in "".join(traceback.format_exception(exc.value)) + + def test_transcription_routes_to_xai_stt(monkeypatch): monkeypatch.delenv("XAI_API_KEY", raising=False) monkeypatch.setattr(litellm, "xai_key", None) From 48ff826cbc4c1f897930d14d6e596543a91dd956 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Thu, 5 Nov 2026 18:50:15 -0800 Subject: [PATCH 34/34] feat(proxy)!: only let clients pick S3 buckets the admin allowed (#47496) * feat(proxy): let admins allow specific S3 buckets and endpoints in request bodies, block bucket settings otherwise Client requests can no longer pick s3_bucket_name, s3_output_bucket_name or allow_legacy_cloud_file_ids unless the value is on general_settings.allowed_client_side_s3_buckets (buckets) or the admin opted in the usual way. s3_endpoint_url, already blocked, can now be allowed per value through general_settings.allowed_client_side_s3_endpoint_urls * refactor(proxy): drive the client-side S3 allowlist from one typed field-to-setting map * fix(proxy): treat a null client-side S3 allowlist in config as empty instead of failing the request --- .../proxy/common_utils/check_batch_cost.py | 13 +- .../proxy/hooks/managed_files.py | 30 +--- litellm/batches/batch_utils.py | 4 +- litellm/batches/main.py | 2 - litellm/files/main.py | 17 --- .../cloud_storage_security.py | 11 +- .../litellm_core_utils/get_litellm_params.py | 22 +-- litellm/llms/bedrock/files/transformation.py | 59 +++----- litellm/proxy/_types.py | 8 ++ litellm/proxy/auth/auth_utils.py | 62 ++++++-- litellm/proxy/batches_endpoints/endpoints.py | 8 -- .../openai_files_endpoints/common_utils.py | 30 ---- .../file_content_streaming_handler.py | 1 - .../openai_files_endpoints/files_endpoints.py | 9 +- litellm/types/router.py | 1 + tests/unit/batches/test_batch_utils.py | 29 ++-- tests/unit/batches/test_main.py | 32 ++--- .../proxy/hooks/test_managed_files.py | 5 - .../proxy/test_managed_files_hook.py | 129 ++++++----------- .../files/test_bedrock_files_handler.py | 132 ++++++++---------- .../test_bedrock_files_transformation.py | 103 ++++++-------- .../files/test_vertex_ai_files_handler.py | 6 +- .../test_vertex_ai_files_transformation.py | 33 +---- tests/unit/proxy/auth/test_auth_utils.py | 118 ++++++++++++++++ .../proxy/batches_endpoints/test_endpoints.py | 5 - .../common_utils/test_check_batch_cost.py | 47 +++---- .../test_files_common_utils.py | 68 --------- .../test_files_endpoint.py | 80 ++++------- .../test_model_based_routing_files_batches.py | 40 ++---- ...st_azure_ad_token_credential_resolution.py | 37 ++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 ++ 31 files changed, 506 insertions(+), 649 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index a31e926256a..4cfc6d4e081 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -8,7 +8,6 @@ import re from collections.abc import Callable, Mapping, Sequence from dataclasses import replace as dataclasses_replace from datetime import datetime, timedelta -from types import MappingProxyType from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tuple, cast from pydantic import TypeAdapter, ValidationError @@ -766,7 +765,6 @@ class CheckBatchCost: credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} _file_content = await afile_content( file_id=cast(str, raw_output_file_id), - _litellm_internal_model_credentials=MappingProxyType(dict(credentials)), **cast(FileContentCallOptions, credentials), ) @@ -867,10 +865,7 @@ class CheckBatchCost: error_file_failed_requests: Final = await count_error_file_failed_requests( response, custom_llm_provider=batch_file_provider, - litellm_params={ - **credentials, - "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)), - }, + litellm_params=credentials, ) batch_result: Final = ( output_file_result @@ -1029,11 +1024,7 @@ class CheckBatchCost: custom_llm_provider=deployment_info.litellm_params.custom_llm_provider, ) cost_provider: Final = cast(BatchFileProvider, llm_provider) - credentials: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} - file_access: Final = { - **credentials, - "_litellm_internal_model_credentials": MappingProxyType(dict(credentials)), - } + file_access: Final = self.llm_router.registry().credentials(model_id, form="with_provider") or {} output_content: Final = ( await fetch_batch_managed_file_content( response.output_file_id, custom_llm_provider=cost_provider, litellm_params=file_access diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 91c5e2c5f39..aac4469be20 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -127,7 +127,6 @@ class _ManagedFileRetrieve(Protocol): self, *, file_id: str, - _litellm_internal_model_credentials: Mapping[str, object] | None = None, **kwargs: Unpack[FileRetrieveCallOptions], ) -> OpenAIFileObject: ... @@ -400,7 +399,7 @@ class _RouterFileCallKwargs(TypedDict, total=False): _MODEL_FILE_ID_MAPPING: Final = TypeAdapter(dict[str, str]) _FILE_UPLOAD_ONLY_FIELDS: Final = frozenset({"file", "purpose", "expires_after", "extra_body"}) -_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id", "_litellm_internal_model_credentials"}) +_FILE_DELETE_RESERVED_FIELDS: Final = frozenset({"model", "file_id"}) @dataclass(frozen=True, slots=True) @@ -459,15 +458,7 @@ async def _delete_provider_file( provider_file_id: str, request_data: Mapping[str, object], ) -> None: - credentials: Final = llm_router.registry().credentials(model_id, form="with_provider") - delete_data: Final = { - **{k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS}, - **( - {"_litellm_internal_model_credentials": MappingProxyType(dict(credentials))} - if credentials is not None - else {} - ), - } + delete_data: Final = {k: v for k, v in request_data.items() if k not in _FILE_DELETE_RESERVED_FIELDS} await llm_router.afile_delete( model=model_id, file_id=provider_file_id, @@ -1160,7 +1151,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): *, provider_file_id: str, call_options: Mapping[str, object], - internal_model_credentials: Mapping[str, object] | None = None, ) -> OpenAIFileObject: """Retrieve provider file details with transient retries and SDK retries disabled.""" retrieve_file: Final = cast(_ManagedFileRetrieve, litellm.afile_retrieve) @@ -1172,14 +1162,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if attempt > 0: await self._sleep(_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS[attempt - 1]) try: - if internal_model_credentials is None: - return await retrieve_file( - file_id=provider_file_id, - **retrieve_options, - ) return await retrieve_file( file_id=provider_file_id, - _litellm_internal_model_credentials=MappingProxyType(dict(internal_model_credentials)), **retrieve_options, ) except Exception as error: @@ -1221,7 +1205,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): self._afile_retrieve_with_retries( provider_file_id=provider_file_id, call_options=router_credentials, - internal_model_credentials=router_credentials, ), timeout=BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS, ) @@ -2866,15 +2849,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): exception_dict = {} for model_id, provider_file_id in specific_model_file_id_mapping.items(): try: - # Cloud-storage providers (e.g. Bedrock S3) validate file ids - # against the deployment's configured bucket, which they only - # trust from this immutable server-side snapshot, never from - # request params. - credentials = llm_router.registry().credentials(model_id, form="with_provider") - if credentials is not None: - data["_litellm_internal_model_credentials"] = cast(Dict, MappingProxyType(dict(credentials))) - else: - data.pop("_litellm_internal_model_credentials", None) return cast( HttpxBinaryResponseContent, await llm_router.afile_content( diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py index d272a235758..ca072368061 100644 --- a/litellm/batches/batch_utils.py +++ b/litellm/batches/batch_utils.py @@ -641,9 +641,11 @@ def extract_file_access_credentials( "bucket_name", "s3_endpoint_url", "s3_region_name", + "s3_bucket_name", + "s3_output_bucket_name", + "allow_legacy_cloud_file_ids", "timeout", "max_retries", - "_litellm_internal_model_credentials", *AWS_CREDENTIAL_KWARGS_KEYS, # A federated deployment holds no api_key, so without these the fetch that reads a # finished batch's output has nothing to authenticate with and its cost is never billed. diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 9b5630d81ad..d88f549b06b 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -22,7 +22,6 @@ from openai.types.batch import BatchRequestCounts import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler from litellm.llms.azure.batches.handler import AzureBatchesAPI @@ -585,7 +584,6 @@ def retrieve_batch( custom_llm_provider=custom_llm_provider, **kwargs, ) - add_trusted_model_credentials_to_litellm_params(litellm_params, kwargs) if litellm_logging_obj is not None: litellm_logging_obj.update_from_kwargs( kwargs=kwargs, diff --git a/litellm/files/main.py b/litellm/files/main.py index c5172bc01ae..d1ddf6c4c4c 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -38,7 +38,6 @@ import litellm from litellm import get_secret_str from litellm.files.streaming import FileContentStreamingResponse from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider -from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure.common_utils import get_azure_credentials @@ -422,10 +421,6 @@ def file_retrieve( ) if provider_config is not None: litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) litellm_params_dict["api_key"] = optional_params.api_key litellm_params_dict["api_base"] = optional_params.api_base @@ -544,10 +539,6 @@ def file_delete( pass optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) ### TIMEOUT LOGIC ### timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 # set timeout for 10 minutes by default @@ -737,10 +728,6 @@ def file_list( ) if provider_config is not None: litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) litellm_params_dict["api_key"] = optional_params.api_key litellm_params_dict["api_base"] = optional_params.api_base @@ -890,10 +877,6 @@ def file_content( try: optional_params: Final = GenericLiteLLMParams(**kwargs) litellm_params_dict: Final = get_litellm_params(**kwargs) - add_trusted_model_credentials_to_litellm_params( - litellm_params_dict=litellm_params_dict, - kwargs=kwargs, - ) ### TIMEOUT LOGIC ### timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 client: Final = kwargs.get("client") diff --git a/litellm/litellm_core_utils/cloud_storage_security.py b/litellm/litellm_core_utils/cloud_storage_security.py index 8e949694881..477775bb423 100644 --- a/litellm/litellm_core_utils/cloud_storage_security.py +++ b/litellm/litellm_core_utils/cloud_storage_security.py @@ -2,12 +2,10 @@ import posixpath import re from collections.abc import Mapping, Sequence from itertools import accumulate, repeat -from types import MappingProxyType from typing import Final from urllib.parse import quote, unquote from litellm._uuid import uuid -from litellm.litellm_core_utils.core_helpers import as_str_mapping VERTEX_AI_MANAGED_GCS_PREFIX: Final = "litellm-vertex-files/" BEDROCK_MANAGED_S3_BATCH_PREFIX: Final = "litellm-bedrock-files-" @@ -19,7 +17,6 @@ BEDROCK_MANAGED_S3_PREFIXES: Final = ( BEDROCK_MANAGED_S3_OUTPUT_PREFIX, ) MANAGED_CLOUD_STORAGE_SCHEMES: Final = ("s3://", "gs://") -_MAPPING_PROXY_TYPE: Final[type] = type(MappingProxyType({})) MAX_FILE_ID_DECODE_PASSES: Final = 8 @@ -138,13 +135,7 @@ def encode_s3_object_key_for_url(object_key: str) -> str: def should_allow_legacy_cloud_file_ids( litellm_params: Mapping[str, object] | None = None, ) -> bool: - value = None - if isinstance(litellm_params, Mapping): - trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials") - if isinstance(trusted_model_credentials, _MAPPING_PROXY_TYPE): - trusted_mapping: Final = as_str_mapping(trusted_model_credentials) - value = trusted_mapping.get("allow_legacy_cloud_file_ids") if trusted_mapping is not None else None - + value: Final = litellm_params.get("allow_legacy_cloud_file_ids") if litellm_params is not None else None if isinstance(value, bool): return value if isinstance(value, str): diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index d0ac5c00513..41c1baffab8 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -1,7 +1,6 @@ import reprlib -from collections.abc import Mapping, MutableMapping +from collections.abc import Mapping from dataclasses import dataclass, fields -from types import MappingProxyType from typing import Final, cast from pydantic import TypeAdapter, ValidationError @@ -61,6 +60,9 @@ OPTIONAL_KWARGS_KEYS: Final = ( "client_side_timeout", "gcs_bucket_name", "bucket_name", + "s3_bucket_name", + "s3_output_bucket_name", + "allow_legacy_cloud_file_ids", "s3_endpoint_url", "s3_region_name", "s3_access_key_id", @@ -292,19 +294,3 @@ def get_litellm_params( litellm_params[key] = kwargs[key] return litellm_params - - -def add_trusted_model_credentials_to_litellm_params( - litellm_params_dict: MutableMapping[str, object], kwargs: Mapping[str, object] -) -> None: - """ - Carry the immutable server-side credential snapshot into litellm_params. - - get_litellm_params has a fixed signature, so callers that need the snapshot to - survive into the logging object and the downstream file read have to re-add it. Only - a MappingProxyType is accepted, since providers resolve trusted configuration such - as a Bedrock file bucket from it and must not read a request-supplied mapping. - """ - trusted_model_credentials: Final = kwargs.get("_litellm_internal_model_credentials") - if isinstance(trusted_model_credentials, MappingProxyType): - litellm_params_dict["_litellm_internal_model_credentials"] = trusted_model_credentials diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 0fd61eb4c07..4d137876890 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -229,15 +229,6 @@ class _S3RequestTarget: request_params: _BedrockS3RequestParams -class _TrustedS3ModelCredentials(LiteLLMBaseModel): - """The S3 buckets the server trusts file ids against, from the deployment snapshot.""" - - model_config = ConfigDict(extra="ignore") - - s3_bucket_name: str | None = None - s3_output_bucket_name: str | None = None - - def extract_s3_uri_from_file_id(file_id: str) -> str: """ Resolve a Bedrock file id to its S3 URI. @@ -261,27 +252,31 @@ def extract_s3_uri_from_file_id(file_id: str) -> str: raise ValueError("file_id must be a managed LiteLLM S3 file id") -_S3_BUCKET_REQUIRED_ERROR: Final = "S3 bucket_name is required. Set 's3_bucket_name' in proxy config or AWS_S3_BUCKET_NAME for Bedrock file content retrieval." +_S3_BUCKET_REQUIRED_ERROR: Final = ( + "S3 bucket_name is required. Pass 's3_bucket_name', set it in proxy config, " + "or set AWS_S3_BUCKET_NAME for Bedrock file content retrieval." +) -def _trusted_s3_model_credentials(litellm_params: Mapping[str, object]) -> _TrustedS3ModelCredentials: - trusted_model_credentials: Final = litellm_params.get("_litellm_internal_model_credentials") - if not isinstance(trusted_model_credentials, MappingProxyType): - return _TrustedS3ModelCredentials() - snapshot: Final[dict[str, object]] = {} - snapshot.update(trusted_model_credentials) # any-ok: untyped snapshot - return _TrustedS3ModelCredentials.model_validate(snapshot) +def _s3_param(litellm_params: Mapping[str, object], key: str, env_var: str) -> str | None: + value: Final = litellm_params.get(key) + return value if isinstance(value, str) and value else os.getenv(env_var) + + +def _input_bucket_name(litellm_params: Mapping[str, object]) -> str | None: + return _s3_param(litellm_params, "s3_bucket_name", "AWS_S3_BUCKET_NAME") + + +def _output_bucket_name(litellm_params: Mapping[str, object]) -> str | None: + return _s3_param(litellm_params, "s3_output_bucket_name", "AWS_S3_OUTPUT_BUCKET_NAME") def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str: """ - Resolve the server-configured S3 bucket for Bedrock file operations. - - Only trusts the immutable server-side credential snapshot or the - environment; never a request-supplied param, since the bucket is what - `validate_managed_cloud_file_id` checks file ids against. + Resolve the S3 bucket for Bedrock file operations: the call's ``s3_bucket_name``, + else ``AWS_S3_BUCKET_NAME``. File ids are validated against this bucket. """ - bucket_name: Final = _trusted_s3_model_credentials(litellm_params).s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME") + bucket_name: Final = _input_bucket_name(litellm_params) if not bucket_name: raise ValueError(_S3_BUCKET_REQUIRED_ERROR) return bucket_name @@ -289,16 +284,13 @@ def get_configured_s3_bucket_name(litellm_params: Mapping[str, object]) -> str: def get_configured_s3_bucket_names(litellm_params: Mapping[str, object]) -> tuple[str, ...]: """ - Resolve the server-configured S3 buckets a Bedrock file id may live in. + Resolve the S3 buckets a Bedrock file id may live in. Bedrock batch outputs land in ``s3_output_bucket_name`` when it differs from - the input bucket, so retrieval validates against both. Same trust rules as - ``get_configured_s3_bucket_name``: only the immutable credential snapshot or - the environment, never a request param. + the input bucket, so retrieval validates against both. """ - trusted: Final = _trusted_s3_model_credentials(litellm_params) - input_bucket: Final = trusted.s3_bucket_name or os.getenv("AWS_S3_BUCKET_NAME") - output_bucket: Final = trusted.s3_output_bucket_name or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME") + input_bucket: Final = _input_bucket_name(litellm_params) + output_bucket: Final = _output_bucket_name(litellm_params) buckets: Final = tuple(dict.fromkeys(bucket for bucket in (input_bucket, output_bucket) if bucket)) if not buckets: raise ValueError(_S3_BUCKET_REQUIRED_ERROR) @@ -514,12 +506,7 @@ def _output_location_still_unlisted(litellm_params: Mapping[str, object]) -> boo def _listing_bucket_name(litellm_params: Mapping[str, object], purpose: str | None) -> str: if purpose != "batch_output": return get_configured_s3_bucket_name(litellm_params) - trusted: Final = _trusted_s3_model_credentials(litellm_params) - return ( - trusted.s3_output_bucket_name - or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME") - or get_configured_s3_bucket_name(litellm_params) - ) + return _output_bucket_name(litellm_params) or get_configured_s3_bucket_name(litellm_params) def _listed_object_created_at(entry: ET.Element) -> int: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d3ac264518e..b82edab85ab 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2813,6 +2813,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): description="How long a blocked source address, or source address and username, stays blocked. Every attempt from a blocked key, right or wrong, is refused with 429 before the password is checked; the block is not extended by refused attempts. Set under `general_settings` in config.yaml. Defaults to 300", ) allowed_routes: list | None = Field(None, description="Proxy API Endpoints you want users to be able to access") + allowed_client_side_s3_buckets: tuple[str, ...] | None = Field( + None, + description="S3 bucket names a client may send as `s3_bucket_name` or `s3_output_bucket_name` in a request body, e.g. on /v1/batches. Any other bucket is rejected unless client-side credentials are allowed", + ) + allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = Field( + None, + description="Exact `s3_endpoint_url` values a client may send in a request body. Any other value is rejected unless client-side credentials are allowed", + ) reject_clientside_metadata_tags: bool | None = Field( None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 161ff2fdfcd..2b5fc6d877e 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -7,10 +7,10 @@ from functools import lru_cache from itertools import accumulate, chain, repeat from logging import Logger from types import MappingProxyType -from typing import Any, Final, Protocol +from typing import Any, Final, Literal, Protocol, TypeAlias from fastapi import HTTPException, Request, status -from pydantic import PositiveInt, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, PositiveInt, TypeAdapter, ValidationError import litellm from litellm import Router, constants, provider_list @@ -442,6 +442,9 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # S3; ``sagemaker_base_url`` redirects all SageMaker traffic; # ``deployment_url`` redirects SAP deployments. "s3_endpoint_url", + "s3_bucket_name", + "s3_output_bucket_name", + "allow_legacy_cloud_file_ids", "sagemaker_base_url", "deployment_url", # NVIDIA Riva fields consumed by the audio-transcription handler @@ -479,6 +482,48 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( ) +_ClientSideAllowlistSetting: TypeAlias = Literal[ + "allowed_client_side_s3_buckets", "allowed_client_side_s3_endpoint_urls" +] + +_PROXY_ALLOWLIST_SETTING_BY_PARAM: Final[Mapping[str, _ClientSideAllowlistSetting]] = MappingProxyType( + { + "s3_bucket_name": "allowed_client_side_s3_buckets", + "s3_output_bucket_name": "allowed_client_side_s3_buckets", + "s3_endpoint_url": "allowed_client_side_s3_endpoint_urls", + } +) + + +class _ClientSideStorageAllowlist(BaseModel): + model_config = ConfigDict(frozen=True) + + allowed_client_side_s3_buckets: tuple[str, ...] | None = None + allowed_client_side_s3_endpoint_urls: tuple[str, ...] | None = None + + def admits(self, setting: _ClientSideAllowlistSetting, value: object) -> bool: + match setting: + case "allowed_client_side_s3_buckets": + return value in (self.allowed_client_side_s3_buckets or ()) + case "allowed_client_side_s3_endpoint_urls": + return value in (self.allowed_client_side_s3_endpoint_urls or ()) + + +def _banned_param_error(param: str) -> ValueError: + setting: Final = _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param) + allowlist_hint: Final = ( + "" if setting is None else f" To allow specific values only, list them in `general_settings.{setting}`." + ) + return ValueError( + f"Rejected Request: {param} is not allowed in request body. " + "Clientside passthrough requires explicit admin opt-in via " + "either `general_settings.allow_client_side_credentials = true` " + "(proxy-wide) or `configurable_clientside_auth_params` on the " + "deployment in your proxy config.yaml. " + f"Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997{allowlist_hint}", + ) + + def _check_banned_params( body: Mapping[str, object], general_settings: dict, @@ -508,6 +553,10 @@ def _check_banned_params( # Proxy-wide opt-in: every banned param is permitted, exit # entirely so the rest of the loop doesn't waste work. return + if ( + setting := _PROXY_ALLOWLIST_SETTING_BY_PARAM.get(param) + ) is not None and _ClientSideStorageAllowlist.model_validate(general_settings).admits(setting, body[param]): + continue if ( _allow_model_level_clientside_configurable_parameters( model=model, @@ -523,14 +572,7 @@ def _check_banned_params( # ``api_base`` with an unallowed ``langfuse_host`` is still # rejected for the second field. continue - raise ValueError( - f"Rejected Request: {param} is not allowed in request body. " - "Clientside passthrough requires explicit admin opt-in via " - "either `general_settings.allow_client_side_credentials = true` " - "(proxy-wide) or `configurable_clientside_auth_params` on the " - "deployment in your proxy config.yaml. " - "Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997", - ) + raise _banned_param_error(param) _FALLBACK_FIELDS: Final[tuple[str, ...]] = ( diff --git a/litellm/proxy/batches_endpoints/endpoints.py b/litellm/proxy/batches_endpoints/endpoints.py index ed737f5202a..1e10d628c1e 100644 --- a/litellm/proxy/batches_endpoints/endpoints.py +++ b/litellm/proxy/batches_endpoints/endpoints.py @@ -48,7 +48,6 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 # BATCH_CREATE_HIDDEN_PARAM, _is_base64_encoded_unified_file_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export add_deployment_model_info, - add_internal_model_credentials, apply_team_provider_credentials, authorize_model_for_key, batch_cost_poller_is_active, @@ -750,13 +749,6 @@ async def retrieve_batch( detail={"error": "LLM Router not initialized. Ensure models added to proxy."}, ) - if unified_batch_id: - add_internal_model_credentials( - data=data, - llm_router=llm_router, - model_id=unified_model_id, - ) - response = await llm_router.aretrieve_batch(**data) set_hidden_param(response, "unified_batch_id", unified_batch_id) if unified_batch_id: diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 18ce18da936..a6eee889fea 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -741,31 +741,6 @@ def apply_team_provider_credentials( prepare_data_with_credentials(data=data, credentials={**credentials, **explicit_request_credentials(data)}) -def add_internal_model_credentials( - data: dict, - llm_router: "Router", - model_id: str | None, -) -> None: - """ - Attach the deployment's immutable server-side credential snapshot to a router-routed - batch call (in-place). - - Cost accounting for a completed batch reads the batch's output file, and the Bedrock - file config resolves its bucket only from this snapshot, never from a request param, - because the bucket is what managed file ids are validated against. Without it that - read fails and the batch's cost is never recorded. - """ - if model_id is None: - return - try: - credentials: Final = llm_router.registry().credentials(model_id, form="with_provider") - except Exception: # noqa: BLE001 # the snapshot only enables cost accounting; a batch whose deployment no longer resolves must still be retrievable - return - if credentials is None: - return - data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials)) - - def add_deployment_model_info( data: dict, llm_router: Optional["Router"], @@ -791,7 +766,6 @@ def prepare_data_with_credentials( data: dict, credentials: dict, file_id: str | None = None, - include_internal_credentials: bool = False, ) -> None: """ Update data dictionary with model credentials (in-place). @@ -800,12 +774,8 @@ def prepare_data_with_credentials( data: Data dictionary to update credentials: Credentials from router file_id: Optional original file_id to set (for decoded file IDs) - include_internal_credentials: Preserve an immutable server-side snapshot - for code paths that must distinguish proxy config from request params. """ data.update(credentials) - if include_internal_credentials: - data["_litellm_internal_model_credentials"] = MappingProxyType(dict(credentials)) data.pop("custom_llm_provider", None) if file_id is not None: diff --git a/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py b/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py index 457af931f56..3b809688d6c 100644 --- a/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py +++ b/litellm/proxy/openai_files_endpoints/file_content_streaming_handler.py @@ -43,7 +43,6 @@ class FileContentStreamingHandler: data=resolved_streaming_data, credentials=credentials, file_id=original_file_id, - include_internal_credentials=True, ) resolved_streaming_data.pop("model", None) resolved_streaming_provider: Final = cast(str, credentials["custom_llm_provider"]) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 9c49c61291f..5ce0ed8b493 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -77,7 +77,6 @@ from litellm.proxy.openai_files_endpoints.batch_guardrails import ( ) from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 # legacy module exports _is_base64_encoded_unified_file_id, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export - add_internal_model_credentials, apply_team_provider_credentials, authorize_model_for_key, deployment_drop_params, @@ -1091,7 +1090,6 @@ async def get_file_content( model: Final = cast(str | None, data.get("model")) if model: - add_internal_model_credentials(data=data, llm_router=llm_router, model_id=model) response = await llm_router.afile_content( **{ "model": model, @@ -1180,7 +1178,6 @@ async def get_file_content( data=data, credentials=credentials, file_id=original_file_id, # Use decoded file ID if from encoded ID - include_internal_credentials=True, ) response = await litellm.afile_content( custom_llm_provider=credentials["custom_llm_provider"], @@ -1354,7 +1351,6 @@ async def get_file( data=data, credentials=credentials, file_id=original_file_id, - include_internal_credentials=True, ) response = await litellm.afile_retrieve( @@ -1568,7 +1564,6 @@ async def delete_file( data=data, credentials=credentials, file_id=original_file_id, - include_internal_credentials=True, ) response = await litellm.afile_delete( @@ -1973,7 +1968,7 @@ async def list_files( ) elif should_route and credentials is not None: # Use model-based routing with credentials from config - prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True) + prepare_data_with_credentials(data=data, credentials=credentials) response = await litellm.afile_list( custom_llm_provider=credentials["custom_llm_provider"], purpose=purpose, @@ -1997,7 +1992,7 @@ async def list_files( user_api_key_dict=user_api_key_dict, operation_context="file list", ) - prepare_data_with_credentials(data=data, credentials=credentials, include_internal_credentials=True) + prepare_data_with_credentials(data=data, credentials=credentials) response = await litellm.afile_list( custom_llm_provider=credentials["custom_llm_provider"], purpose=purpose, diff --git a/litellm/types/router.py b/litellm/types/router.py index 3363d113fc7..a9f28fbc673 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -427,6 +427,7 @@ class CredentialLiteLLMParams(LiteLLMBaseModel): ## OBJECT STORAGE (files / batches) ## gcs_bucket_name: str | None = None bucket_name: str | None = None + allow_legacy_cloud_file_ids: bool | None = None ## AWS BEDROCK / SAGEMAKER ## aws_access_key_id: str | None = None diff --git a/tests/unit/batches/test_batch_utils.py b/tests/unit/batches/test_batch_utils.py index 0f85f1e3993..896fc9d975a 100644 --- a/tests/unit/batches/test_batch_utils.py +++ b/tests/unit/batches/test_batch_utils.py @@ -16,7 +16,6 @@ deterministic stand-ins so the arithmetic under test is the only variable. import asyncio, json, time import logging -from types import MappingProxyType from typing import Final, Literal import httpx @@ -1692,17 +1691,21 @@ async def test_calculate_batch_cost_and_usage_anthropic_end_to_end(): assert result.models == ["claude-sonnet-4-5"] -def test_extract_credentials_forwards_the_trusted_model_credential_snapshot(): - """Bedrock resolves a batch's output bucket only from the immutable server-side - snapshot, never from a request param, so cost accounting on the retrieve path cannot - read the output file unless this key is forwarded. Without it the accounting raises - "S3 bucket_name is required" for a bucket the deployment has configured, and the - batch's cost is never recorded.""" - snapshot = MappingProxyType({"s3_bucket_name": "configured-bucket", "aws_region_name": "us-east-1"}) +def test_extract_credentials_forwards_the_deployment_bucket_settings() -> None: + """Cost accounting on the retrieve path reads the batch's output file from the + deployment's Bedrock buckets. Without these keys the read raises "S3 bucket_name is + required" for a bucket the deployment has configured, and the batch's cost is never + recorded.""" + params: Final = { + "s3_bucket_name": "configured-bucket", + "s3_output_bucket_name": "output-bucket", + "allow_legacy_cloud_file_ids": True, + "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", + } - credentials = bu._extract_file_access_credentials({"_litellm_internal_model_credentials": snapshot}) + credentials: Final = bu._extract_file_access_credentials(params) - assert credentials["_litellm_internal_model_credentials"] is snapshot + assert credentials == {key: value for key, value in params.items() if key != "model"} def test_extract_credentials_forwards_the_deployment_aws_credentials(): @@ -1736,8 +1739,6 @@ async def test_output_file_content_bedrock_reads_with_deployment_aws_credentials return type("R", (), {"content": b""})() monkeypatch.setattr(files_main, "afile_content", fake_afile_content) - snapshot = MappingProxyType({"s3_bucket_name": "configured-bucket", "aws_region_name": "us-west-2"}) - await bu._fetch_batch_output_file_content( _batch("s3://configured-bucket/litellm-batch-outputs/job-1/out.jsonl.out"), custom_llm_provider="bedrock", @@ -1746,7 +1747,7 @@ async def test_output_file_content_bedrock_reads_with_deployment_aws_credentials "aws_secret_access_key": "secret-deployment", "aws_session_token": "token-deployment", "aws_region_name": "us-west-2", - "_litellm_internal_model_credentials": snapshot, + "s3_bucket_name": "configured-bucket", "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", }, ) @@ -1757,7 +1758,7 @@ async def test_output_file_content_bedrock_reads_with_deployment_aws_credentials assert captured["aws_secret_access_key"] == "secret-deployment" assert captured["aws_session_token"] == "token-deployment" assert captured["aws_region_name"] == "us-west-2" - assert captured["_litellm_internal_model_credentials"] is snapshot + assert captured["s3_bucket_name"] == "configured-bucket" assert "model" not in captured diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index 5ffe6558dc3..8cd74c7d83a 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -732,34 +732,30 @@ def test_resolve_timeout__httpx_timeout_returns_float_read(): assert resolved == 99.0 -def test_retrieve__forwards_trusted_model_credentials_into_litellm_params(seams): +def test_retrieve__keeps_the_deployment_bucket_settings_in_litellm_params(seams: Seams) -> None: """The batch's cost is computed by reading its output file after the retrieve, and - Bedrock resolves that bucket only from this immutable snapshot. get_litellm_params has - a fixed signature that drops it, so without re-adding it here the snapshot never - reaches the logging object and cost accounting fails on a bucket that is configured.""" - snapshot = MappingProxyType({"s3_bucket_name": "configured-bucket"}) + that read finds the Bedrock bucket in these litellm_params. Dropping them fails cost + accounting on a bucket the deployment has configured.""" logging_obj = MagicMock() bm.retrieve_batch( batch_id="batch-1", custom_llm_provider="openai", litellm_logging_obj=logging_obj, - _litellm_internal_model_credentials=snapshot, + s3_bucket_name="configured-bucket", + s3_output_bucket_name="output-bucket", + allow_legacy_cloud_file_ids=True, ) litellm_params = logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"] - assert litellm_params["_litellm_internal_model_credentials"] is snapshot - - -def test_retrieve__omits_trusted_model_credentials_when_not_supplied(seams): - """A retrieve with no snapshot must not invent an empty one, which would read as a - configured bucket of nothing.""" - logging_obj = MagicMock() - - bm.retrieve_batch(batch_id="batch-1", custom_llm_provider="openai", litellm_logging_obj=logging_obj) - - litellm_params = logging_obj.update_from_kwargs.call_args.kwargs["litellm_params"] - assert "_litellm_internal_model_credentials" not in litellm_params + assert { + key: litellm_params[key] + for key in ("s3_bucket_name", "s3_output_bucket_name", "allow_legacy_cloud_file_ids") + } == { + "s3_bucket_name": "configured-bucket", + "s3_output_bucket_name": "output-bucket", + "allow_legacy_cloud_file_ids": True, + } # =========================================================================== # diff --git a/tests/unit/enterprise/proxy/hooks/test_managed_files.py b/tests/unit/enterprise/proxy/hooks/test_managed_files.py index a8dc506f306..747c09b9108 100644 --- a/tests/unit/enterprise/proxy/hooks/test_managed_files.py +++ b/tests/unit/enterprise/proxy/hooks/test_managed_files.py @@ -1980,7 +1980,6 @@ async def test_store_batch_output_file_retries_model_name_provider_fetch(): assert all( call.kwargs["custom_llm_provider"] == "bedrock" and call.kwargs["max_retries"] == 0 - and "_litellm_internal_model_credentials" not in call.kwargs for call in retrieve.call_args_list ) assert sleep_calls == [0.5, 1.0] @@ -2022,7 +2021,6 @@ async def test_store_batch_output_file_marks_model_name_fallback_after_four_tran assert all( call.kwargs["custom_llm_provider"] == "bedrock" and call.kwargs["max_retries"] == 0 - and "_litellm_internal_model_credentials" not in call.kwargs for call in retrieve.call_args_list ) assert sleep_calls == [0.5, 1.0, 2.0] @@ -3656,7 +3654,6 @@ async def test_afile_retrieve_refreshes_marked_fallback_without_router_from_mode retrieve.assert_awaited_once() assert retrieve.await_args.kwargs["custom_llm_provider"] == "bedrock" assert retrieve.await_args.kwargs["max_retries"] == 0 - assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs updated_row: Final = managed_file_table.rows["unified-output"] assert updated_row.file_object == response assert updated_row.model_mappings == {"bedrock/model-x": "provider-output"} @@ -3700,7 +3697,6 @@ async def test_afile_retrieve_uses_model_name_when_router_does_not_know_deployme ) assert retrieve.await_args.kwargs["custom_llm_provider"] == "bedrock" assert retrieve.await_args.kwargs["max_retries"] == 0 - assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs @pytest.mark.asyncio @@ -3828,7 +3824,6 @@ async def test_afile_retrieve_case3_uses_default_provider_for_unknown_router_dep assert retrieve.await_args.kwargs["file_id"] == "provider-output" assert retrieve.await_args.kwargs["max_retries"] == 0 assert "custom_llm_provider" not in retrieve.await_args.kwargs - assert "_litellm_internal_model_credentials" not in retrieve.await_args.kwargs @pytest.mark.asyncio diff --git a/tests/unit/enterprise/proxy/test_managed_files_hook.py b/tests/unit/enterprise/proxy/test_managed_files_hook.py index 942433b39fb..9ec2a67da0d 100644 --- a/tests/unit/enterprise/proxy/test_managed_files_hook.py +++ b/tests/unit/enterprise/proxy/test_managed_files_hook.py @@ -1165,45 +1165,56 @@ async def test_should_skip_non_file_unified_id_on_output_file_id(): @pytest.mark.asyncio -async def test_afile_content_passes_trusted_model_credentials_to_router(): - """ - afile_content must hand the deployment's credential snapshot to the router - call as an immutable server-side mapping. Cloud-storage providers (Bedrock - S3) validate file ids against the bucket in that snapshot, so without it - unified-id content retrieval only works when AWS_S3_BUCKET_NAME is set. - """ - from types import MappingProxyType +async def test_afile_content_reads_from_the_deployment_bucket(monkeypatch: pytest.MonkeyPatch) -> None: + """Unified-id content retrieval must validate and fetch the s3:// file against the + deployment's s3_bucket_name, with no AWS_S3_BUCKET_NAME env set.""" + import httpx + import respx + + import litellm + from litellm import Router + + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + router = Router( + model_list=[ + { + "model_name": "bedrock-claude", + "litellm_params": { + "model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", + "aws_region_name": "us-west-2", + "s3_bucket_name": "my-bucket", + }, + "model_info": {"id": "model-123"}, + } + ], + num_retries=0, + ) managed_files = _make_managed_files_instance() unified_file_id = "unified-file-id" s3_uri = "s3://my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock( - return_value={unified_file_id: {"model-123": s3_uri}} - ) + managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) - mock_router = MagicMock() - mock_router.registry.return_value.credentials = MagicMock( - return_value={ - "custom_llm_provider": "bedrock", - "s3_bucket_name": "my-bucket", - "aws_region_name": "us-west-2", - } - ) - mock_router.afile_content = AsyncMock(return_value=MagicMock()) + with respx.mock: + route = respx.get( + "https://s3.us-west-2.amazonaws.com/my-bucket/litellm-batch-outputs/job-123/input.jsonl.out" + ).mock(return_value=httpx.Response(200, content=b'{"recordId":"req-1"}')) - await managed_files.afile_content( - file_id=unified_file_id, - litellm_parent_otel_span=None, - llm_router=mock_router, - ) + response = await managed_files.afile_content( + file_id=unified_file_id, + litellm_parent_otel_span=None, + llm_router=router, + ) - call_kwargs = mock_router.afile_content.call_args.kwargs - assert call_kwargs["model"] == "model-123" - assert call_kwargs["file_id"] == s3_uri - trusted_credentials = call_kwargs["_litellm_internal_model_credentials"] - assert isinstance(trusted_credentials, MappingProxyType) - assert trusted_credentials["s3_bucket_name"] == "my-bucket" + assert route.calls[0].request.headers["Authorization"].startswith("AWS4-HMAC-SHA256") + assert response.content == b'{"recordId":"req-1"}' def _managed_deletion_file_id(provider_file_id): @@ -1273,7 +1284,6 @@ async def test_afile_delete_bedrock_uses_deployment_bucket_and_signed_s3_delete( file_id=unified_file_id, litellm_parent_otel_span=None, llm_router=router, - _litellm_internal_model_credentials={"s3_bucket_name": "request-bucket"}, ) assert len(route.calls) == 1 @@ -1299,7 +1309,6 @@ async def test_afile_delete_returns_managed_id_for_stored_provider_output(): file_id=unified_file_id, litellm_parent_otel_span=None, llm_router=router, - _litellm_internal_model_credentials={"s3_bucket_name": "request-bucket"}, ) assert response.id == unified_file_id @@ -1928,45 +1937,6 @@ async def test_post_call_hook_leaves_litellm_executed_batches_untouched(llm_batc assert response.id == original_id -@pytest.mark.asyncio -async def test_afile_delete_passes_trusted_model_credentials_to_router(): - """ - afile_delete must hand the deployment's credential snapshot to the router - call, since Bedrock validates the s3:// file id against the bucket in it. - """ - from types import MappingProxyType - - managed_files = _make_managed_files_instance() - unified_file_id = "unified-file-id" - s3_uri = "s3://my-bucket/litellm-bedrock-files/job-123/input.jsonl" - managed_files.get_unified_file_id = AsyncMock(return_value=None) - managed_files.get_model_file_id_mapping = AsyncMock(return_value={unified_file_id: {"model-123": s3_uri}}) - managed_files.delete_unified_file_id = AsyncMock(return_value=_make_file_object(unified_file_id)) - - mock_router = MagicMock() - mock_router.registry.return_value.credentials = MagicMock( - return_value={ - "custom_llm_provider": "bedrock", - "s3_bucket_name": "my-bucket", - "aws_region_name": "us-west-2", - } - ) - mock_router.afile_delete = AsyncMock(return_value=MagicMock()) - - await managed_files.afile_delete( - file_id=unified_file_id, - litellm_parent_otel_span=None, - llm_router=mock_router, - ) - - call_kwargs = mock_router.afile_delete.call_args.kwargs - assert call_kwargs["model"] == "model-123" - assert call_kwargs["file_id"] == s3_uri - trusted_credentials = call_kwargs["_litellm_internal_model_credentials"] - assert isinstance(trusted_credentials, MappingProxyType) - assert trusted_credentials["s3_bucket_name"] == "my-bucket" - - @pytest.mark.asyncio async def test_afile_delete_bedrock_unified_id_end_to_end(monkeypatch): """ @@ -2291,7 +2261,7 @@ class _FakeFilesRouter: self.delete_gate = delete_gate self.provider = provider self.upload_count = 0 - self.deletes: tuple[tuple[str, str, Mapping[str, object], Mapping[str, str]], ...] = () + self.deletes: tuple[tuple[str, str, Mapping[str, object]], ...] = () self.deletes_marked_internal: tuple[bool, ...] = () self.delete_parent_request_ids: tuple[str | None, ...] = () @@ -2328,14 +2298,13 @@ class _FakeFilesRouter: model: str, file_id: str, litellm_metadata: Mapping[str, object], - _litellm_internal_model_credentials: Mapping[str, str], litellm_call_id: str | None = None, ) -> None: if self.delete_gate is not None: await self.delete_gate.wait() if isinstance(litellm_metadata, dict): litellm_metadata.update({"model_info": {"id": model}}) - self.deletes = (*self.deletes, (model, file_id, litellm_metadata, _litellm_internal_model_credentials)) + self.deletes = (*self.deletes, (model, file_id, litellm_metadata)) self.deletes_marked_internal = (*self.deletes_marked_internal, is_internal_call.get()) self.delete_parent_request_ids = (*self.delete_parent_request_ids, internal_parent_request_id.get()) if file_id in self.gone_file_ids: @@ -2399,15 +2368,7 @@ async def test_acreate_file_keeps_the_upload_the_managed_id_carries_when_two_tar assert carried == ("wildcard-dep", "file-upload-1") assert model_mappings == {"wildcard-dep": "file-upload-1"} assert flat_ids == ["file-upload-1"] - assert router.deletes == ( - ( - "wildcard-dep", - "file-upload-2", - _REQUEST_METADATA, - {"custom_llm_provider": "openai", "api_key": "key-of-wildcard-dep"}, - ), - ) - assert isinstance(router.deletes[0][3], MappingProxyType) + assert router.deletes == (("wildcard-dep", "file-upload-2", _REQUEST_METADATA),) @pytest.mark.asyncio diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py index 5c078affffc..2b76b54e905 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_handler.py @@ -1,12 +1,15 @@ import base64 import os from types import MappingProxyType -from unittest.mock import MagicMock, patch +from typing import Final +from unittest.mock import patch +import httpx import pytest import litellm.files.main as files_main from litellm.llms.bedrock.files.handler import BedrockFilesHandler +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import SpecialEnums @@ -123,88 +126,67 @@ class TestBedrockFilesHandler: configured_bucket_name="safe-bucket", ) - def test_should_not_trust_request_s3_bucket_name_for_expected_bucket(self): - with patch.dict(os.environ, {"AWS_S3_BUCKET_NAME": "safe-bucket"}): - assert ( - self.handler._get_configured_s3_bucket_name( - {"s3_bucket_name": "attacker-bucket"} - ) - == "safe-bucket" - ) + def test_call_s3_bucket_name_wins_over_env_bucket(self): + with patch.dict(os.environ, {"AWS_S3_BUCKET_NAME": "env-bucket"}): + assert self.handler._get_configured_s3_bucket_name({"s3_bucket_name": "call-bucket"}) == "call-bucket" - def test_should_trust_proxy_config_s3_bucket_name_for_expected_bucket(self): - trusted_credentials = MappingProxyType({"s3_bucket_name": "safe-bucket"}) + def test_env_bucket_is_used_when_the_call_names_none(self): + with patch.dict(os.environ, {"AWS_S3_BUCKET_NAME": "env-bucket"}): + assert self.handler._get_configured_s3_bucket_name({}) == "env-bucket" - with patch.dict(os.environ, {}, clear=True): - assert ( - self.handler._get_configured_s3_bucket_name( - { - "s3_bucket_name": "attacker-bucket", - "_litellm_internal_model_credentials": trusted_credentials, - } - ) - == "safe-bucket" - ) - - def test_should_not_trust_user_supplied_internal_credentials_dict(self): + def test_missing_bucket_in_call_and_env_is_rejected(self): with patch.dict(os.environ, {}, clear=True): with pytest.raises(ValueError, match="S3 bucket_name is required"): - self.handler._get_configured_s3_bucket_name( - { - "_litellm_internal_model_credentials": { - "s3_bucket_name": "attacker-bucket" - } - } - ) - - def test_should_require_server_s3_bucket_name(self): - with patch.dict(os.environ, {}, clear=True): - with pytest.raises(ValueError, match="S3 bucket_name is required"): - self.handler._get_configured_s3_bucket_name( - {"s3_bucket_name": "attacker-bucket"} - ) + self.handler._get_configured_s3_bucket_name({}) -def test_should_forward_trusted_model_credentials_to_bedrock_provider_config(): - trusted_credentials = MappingProxyType({"s3_bucket_name": "safe-bucket"}) - mock_response = MagicMock() +def _echo_request_url_client() -> HTTPHandler: + return HTTPHandler( + client=httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=str(request.url).encode())) + ) + ) - with patch.object( - files_main.base_llm_http_handler, - "retrieve_file_content", - return_value=mock_response, - ) as mock_retrieve_file_content: - response = files_main.file_content( - file_id="s3://safe-bucket/litellm-bedrock-files/file.jsonl", + +_SDK_AWS_CREDENTIALS: Final = MappingProxyType( + { + "aws_access_key_id": "AKIASDKCALLER", + "aws_secret_access_key": "sdk-caller-secret", + "aws_region_name": "us-west-2", + } +) + + +def test_sdk_file_content_reads_from_the_bucket_passed_to_the_call(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + + response: Final = files_main.file_content( + file_id="s3://sdk-bucket/litellm-bedrock-files/batch.jsonl", + custom_llm_provider="bedrock", + s3_bucket_name="sdk-bucket", + client=_echo_request_url_client(), + **_SDK_AWS_CREDENTIALS, + ) + + assert httpx.URL(response.content.decode()).path == "/sdk-bucket/litellm-bedrock-files/batch.jsonl" + + +def test_sdk_file_content_rejects_a_file_outside_the_bucket_passed_to_the_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + + with pytest.raises(Exception, match="configured storage bucket"): + files_main.file_content( + file_id="s3://other-bucket/litellm-bedrock-files/batch.jsonl", custom_llm_provider="bedrock", - _litellm_internal_model_credentials=trusted_credentials, + s3_bucket_name="sdk-bucket", + client=_echo_request_url_client(), + **_SDK_AWS_CREDENTIALS, ) - assert response is mock_response - litellm_params = mock_retrieve_file_content.call_args.kwargs["litellm_params"] - assert litellm_params["_litellm_internal_model_credentials"] is trusted_credentials - assert "s3_bucket_name" not in litellm_params - - -def test_should_forward_trusted_model_credentials_to_retrieve_provider_config(): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) - mock_response = MagicMock() - - with patch.object( - files_main.base_llm_http_handler, - "retrieve_file", - return_value=mock_response, - ) as mock_retrieve_file: - response = files_main.file_retrieve( - file_id="gs://safe-bucket/private/file.jsonl", - custom_llm_provider="vertex_ai", - _litellm_internal_model_credentials=trusted_credentials, - ) - - assert response is mock_response - litellm_params = mock_retrieve_file.call_args.kwargs["litellm_params"] - assert litellm_params["_litellm_internal_model_credentials"] is trusted_credentials - @pytest.mark.asyncio async def test_afile_content_assumes_role_with_external_id(monkeypatch): @@ -249,7 +231,7 @@ async def test_afile_content_assumes_role_with_external_id(monkeypatch): return FakeS3Client() optional_params = { - "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "s3_bucket_name": "safe-bucket", "aws_region_name": "us-east-1", "aws_access_key_id": "AKIAFILESDOWNLOADCALLER", "aws_secret_access_key": "pod-caller-secret", @@ -285,7 +267,7 @@ async def test_afile_content_builds_the_s3_client_with_the_s3_pair_when_it_diffe return {"Body": FakeS3Body()} optional_params = { - "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "s3_bucket_name": "safe-bucket", "aws_region_name": "us-east-1", "aws_access_key_id": "AKIABEDROCKONLY", "aws_secret_access_key": "bedrock-only-secret", diff --git a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py index a9edbd69ee3..93bda5b444d 100644 --- a/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py +++ b/tests/unit/llms/bedrock/files/test_bedrock_files_transformation.py @@ -9,7 +9,6 @@ from contextlib import AsyncExitStack, closing from datetime import datetime, timezone from io import BytesIO from pathlib import Path -from types import MappingProxyType from typing import Final from unittest.mock import MagicMock from urllib.parse import unquote, urlparse @@ -2351,7 +2350,7 @@ class TestBedrockFileContentTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params: Final = _trusted_bucket_snapshot( + litellm_params: Final = _deployment_bucket_params( s3_bucket_name="in-bucket/pfx", s3_output_bucket_name="out-bucket/outpfx", ) @@ -2541,7 +2540,7 @@ class TestBedrockFileContentTransformation: def test_transform_file_content_request_requires_configured_bucket( self, monkeypatch ): - """Without a server-configured bucket (env or snapshot), the request must fail + """Without a bucket (on the call or in env), the request must fail before any S3 call rather than guessing a bucket from the file id.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig @@ -2566,30 +2565,24 @@ class TestBedrockFileContentTransformation: litellm_params=self._litellm_params(), ) - def _trusted(self, **deployment_litellm_params) -> dict: - """Build the trusted snapshot the way the proxy does: deployment - litellm_params funneled through ``CredentialLiteLLMParams`` (the strict - allowlist ``get_deployment_credentials_with_provider`` applies) before - retrieval ever sees them. Injecting a raw ``MappingProxyType`` would - bypass that filter and hide whether a bucket field actually survives - into the snapshot in production.""" - from types import MappingProxyType - + def _deployment_params(self, **deployment_litellm_params) -> dict: + """Deployment litellm_params funneled through ``CredentialLiteLLMParams`` + (the strict allowlist ``get_deployment_credentials_with_provider`` applies) + the way the proxy hands them to retrieval, so a bucket field dropped from + that allowlist fails these tests.""" from litellm.types.router import CredentialLiteLLMParams - snapshot = CredentialLiteLLMParams(**deployment_litellm_params).model_dump( + credentials = CredentialLiteLLMParams(**deployment_litellm_params).model_dump( exclude_none=True ) - params = self._litellm_params() - params["_litellm_internal_model_credentials"] = MappingProxyType(snapshot) - return params + return {**self._litellm_params(), **credentials} def test_retrieves_from_distinct_output_bucket(self, monkeypatch): """Batch outputs can land in a separate s3_output_bucket_name. Retrieval must validate the file id against the output bucket too, not just the input bucket, or the very outputs the feature serves are unreachable. - The snapshot is built through the production credential filter, so this - fails if s3_output_bucket_name is dropped from that allowlist.""" + The params go through the production credential filter, so this fails + if s3_output_bucket_name is dropped from that allowlist.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) @@ -2600,7 +2593,7 @@ class TestBedrockFileContentTransformation: "file_id": "s3://out-bucket/litellm-batch-outputs/job/in.jsonl.out" }, optional_params={}, - litellm_params=self._trusted( + litellm_params=self._deployment_params( s3_bucket_name="in-bucket", s3_output_bucket_name="out-bucket" ), ) @@ -2611,8 +2604,8 @@ class TestBedrockFileContentTransformation: ) def test_output_bucket_falls_back_to_env(self, monkeypatch): - """The output bucket resolves from AWS_S3_OUTPUT_BUCKET_NAME when not in - the trusted snapshot, mirroring the input-bucket env fallback.""" + """The output bucket resolves from AWS_S3_OUTPUT_BUCKET_NAME when the + call has none, mirroring the input-bucket env fallback.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.setenv("AWS_S3_BUCKET_NAME", "in-bucket") @@ -2644,7 +2637,7 @@ class TestBedrockFileContentTransformation: "file_id": "s3://in-bucket/litellm-batch-outputs/job/in.jsonl.out" }, optional_params={}, - litellm_params=self._trusted( + litellm_params=self._deployment_params( s3_bucket_name="in-bucket", s3_output_bucket_name="out-bucket" ), ) @@ -2669,7 +2662,7 @@ class TestBedrockFileContentTransformation: "file_id": "s3://other-bucket/litellm-batch-outputs/job/x.jsonl.out" }, optional_params={}, - litellm_params=self._trusted( + litellm_params=self._deployment_params( s3_bucket_name="in-bucket", s3_output_bucket_name="out-bucket" ), ) @@ -2694,22 +2687,16 @@ class TestBedrockFileContentTransformation: litellm_params=self._litellm_params(), ) - def test_bucket_resolved_from_trusted_model_credentials(self, monkeypatch): - """Per-model s3_bucket_name must be honored via the server-side credential snapshot.""" - from types import MappingProxyType - + def test_bucket_resolved_from_call_s3_bucket_name(self, monkeypatch): + """An s3_bucket_name on the call is honored without AWS_S3_BUCKET_NAME.""" from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) - litellm_params = self._litellm_params() - litellm_params["_litellm_internal_model_credentials"] = MappingProxyType( - {"s3_bucket_name": "my-bucket"} - ) url, _ = BedrockFilesConfig().transform_file_content_request( file_content_request={"file_id": self.S3_URI}, optional_params={}, - litellm_params=litellm_params, + litellm_params={**self._litellm_params(), "s3_bucket_name": "my-bucket"}, ) assert url == self.EXPECTED_URL @@ -3226,13 +3213,13 @@ def _bedrock_s3_params() -> dict: } -def _trusted_bucket_snapshot(**deployment_litellm_params) -> dict: - from types import MappingProxyType - +def _deployment_bucket_params(**deployment_litellm_params: object) -> dict[str, object]: from litellm.types.router import CredentialLiteLLMParams - snapshot = CredentialLiteLLMParams(**deployment_litellm_params).model_dump(exclude_none=True) - return {**_bedrock_s3_params(), "_litellm_internal_model_credentials": MappingProxyType(snapshot)} + credentials: Final = CredentialLiteLLMParams.model_validate(deployment_litellm_params).model_dump( + exclude_none=True + ) + return {**_bedrock_s3_params(), **credentials} class TestBedrockFileDeletionTransformation: @@ -3287,7 +3274,7 @@ class TestBedrockFileDeletionTransformation: assert url == self.EXPECTED_URL - def test_transform_delete_file_request_uses_trusted_snapshot_bucket(self, monkeypatch): + def test_transform_delete_file_request_uses_call_bucket(self, monkeypatch): from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) @@ -3295,7 +3282,7 @@ class TestBedrockFileDeletionTransformation: url, _ = BedrockFilesConfig().transform_delete_file_request( file_id=self.S3_URI, optional_params={}, - litellm_params=_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + litellm_params=_deployment_bucket_params(s3_bucket_name="my-bucket"), ) assert url == self.EXPECTED_URL @@ -3595,7 +3582,7 @@ class TestBedrockFileListTransformation: "GET", f"{url}?list-type=2&prefix=LLM%20AI%20Projects%2Flitellm-b", signed_headers ) - def test_transform_list_files_request_uses_trusted_snapshot_bucket(self, monkeypatch): + def test_transform_list_files_request_uses_call_bucket(self, monkeypatch): from litellm.llms.bedrock.files.transformation import BedrockFilesConfig monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) @@ -3603,7 +3590,7 @@ class TestBedrockFileListTransformation: url, params = BedrockFilesConfig().transform_list_files_request( purpose=None, optional_params={}, - litellm_params=_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + litellm_params=_deployment_bucket_params(s3_bucket_name="my-bucket"), ) assert url == self.BUCKET_URL @@ -3696,7 +3683,7 @@ class TestBedrockFileListTransformation: assert _sent_signature(request.headers) == _s3_signature_for("GET", str(request.url), request.headers) assert [file.id for file in files] == list(self.BATCH_IDS) - def test_file_list_uses_trusted_snapshot_bucket_without_env(self, monkeypatch): + def test_file_list_uses_call_bucket_without_env(self, monkeypatch): import httpx import respx @@ -3711,7 +3698,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) assert route.called @@ -3766,7 +3753,7 @@ class TestBedrockFileListTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params = _trusted_bucket_snapshot( + litellm_params = _deployment_bucket_params( s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket/team-a" ) @@ -3852,7 +3839,7 @@ class TestBedrockFileListTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params = _trusted_bucket_snapshot(s3_output_bucket_name="my-output-bucket") + litellm_params = _deployment_bucket_params(s3_output_bucket_name="my-output-bucket") url, params = BedrockFilesConfig().transform_list_files_request( purpose="batch_output", optional_params={}, litellm_params=litellm_params @@ -3879,7 +3866,7 @@ class TestBedrockFileListTransformation: raw_response=httpx.Response(200, content=self.OUTPUT_BUCKET_LISTING), logging_obj=MagicMock(), litellm_params={ - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), LIST_FILES_PURPOSE_PARAM: "batch_output", }, ) @@ -3903,7 +3890,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", purpose="batch_output", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), ) assert route.called @@ -3930,7 +3917,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), ) assert (input_route.call_count, output_route.call_count) == (1, 1) @@ -3962,7 +3949,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-output-bucket"), ) requested_urls = [str(call.request.url) for call in respx.calls] @@ -3975,7 +3962,7 @@ class TestBedrockFileListTransformation: assert [file.id for file in files] == [*self.PAGED_IDS, self.OUTPUT_BUCKET_ID] @pytest.mark.parametrize( - ("purpose", "bucket_snapshot"), + ("purpose", "bucket_params"), [ pytest.param(None, {"s3_bucket_name": "my-bucket"}, id="outputs-share-the-input-bucket"), pytest.param( @@ -3986,7 +3973,7 @@ class TestBedrockFileListTransformation: ], ) def test_file_list_leaves_the_output_bucket_alone_unless_an_unfiltered_list_needs_it( - self, monkeypatch, purpose, bucket_snapshot + self, monkeypatch, purpose, bucket_params ): import httpx import respx @@ -4003,7 +3990,7 @@ class TestBedrockFileListTransformation: ) files = litellm.file_list( - custom_llm_provider="bedrock", purpose=purpose, **_trusted_bucket_snapshot(**bucket_snapshot) + custom_llm_provider="bedrock", purpose=purpose, **_deployment_bucket_params(**bucket_params) ) assert (input_route.call_count, output_route.call_count) == (1, 0) @@ -4019,7 +4006,7 @@ class TestBedrockFileListTransformation: monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) - litellm_params = _trusted_bucket_snapshot(s3_bucket_name="my-bucket", s3_output_bucket_name="my-bucket/out") + litellm_params = _deployment_bucket_params(s3_bucket_name="my-bucket", s3_output_bucket_name="my-bucket/out") config = BedrockFilesConfig() config.transform_list_files_request(purpose=None, optional_params={}, litellm_params=litellm_params) litellm_params.pop(S3_SIGNED_REQUEST_HEADERS_PARAM) @@ -4130,7 +4117,7 @@ class TestBedrockFileListTransformation: custom_llm_provider="bedrock", purpose="batch", timeout=12, - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_paged_listing(first_page, last_page, files) @@ -4152,7 +4139,7 @@ class TestBedrockFileListTransformation: custom_llm_provider="bedrock", purpose="batch", timeout=12, - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_paged_listing(first_page, last_page, files) @@ -4209,7 +4196,7 @@ class TestBedrockFileListTransformation: files = litellm.file_list( custom_llm_provider="bedrock", purpose="batch", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_capped_listing(route, files) @@ -4230,7 +4217,7 @@ class TestBedrockFileListTransformation: files = await litellm.afile_list( custom_llm_provider="bedrock", purpose="batch", - **_trusted_bucket_snapshot(s3_bucket_name="my-bucket"), + **_deployment_bucket_params(s3_bucket_name="my-bucket"), ) self._assert_capped_listing(route, files) @@ -4321,7 +4308,7 @@ def test_transform_file_content_request_signs_with_the_s3_pair_from_litellm_para litellm_params = { **_SPLIT_IDENTITY_PARAMS, - "_litellm_internal_model_credentials": MappingProxyType({"s3_bucket_name": "safe-bucket"}), + "s3_bucket_name": "safe-bucket", } BedrockFilesConfig().transform_file_content_request( file_content_request={"file_id": "s3://safe-bucket/litellm-bedrock-files-model-id-abc.jsonl"}, diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py index 39310836bf2..7d60de73718 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_handler.py @@ -3,7 +3,6 @@ Test Vertex AI files handler functionality """ import re -from types import MappingProxyType import pytest from unittest.mock import AsyncMock, patch @@ -62,15 +61,14 @@ class TestVertexAIFilesHandler: configured_bucket_name="test-bucket", ) - def test_extract_bucket_and_object_from_file_id_allows_trusted_legacy_flag(self): + def test_extract_bucket_and_object_from_file_id_allows_legacy_flag_on_the_call(self): file_id = "gs%3A%2F%2Ftest-bucket%2Ftest-file.txt" - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) bucket_name, object_path = self.handler._extract_bucket_and_object_from_file_id( file_id=file_id, configured_bucket_name="test-bucket", litellm_params={ - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 842500a7b27..c6b6bca2014 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -6,8 +6,8 @@ Includes tests for Vertex AI batch output transformation to OpenAI format. import json import urllib.parse from collections.abc import Mapping, Sequence -from typing import Final from types import MappingProxyType +from typing import Final from urllib.parse import parse_qs, urlparse import httpx @@ -72,46 +72,24 @@ class TestParseGcsUri: litellm_params={"gcs_bucket_name": "my-bucket"}, ) - def test_should_reject_request_supplied_legacy_flag(self, config): - with pytest.raises(ValueError, match="LiteLLM-managed"): - config._parse_gcs_uri( - "gs://my-bucket/private/object.txt", - litellm_params={ - "gcs_bucket_name": "my-bucket", - "allow_legacy_cloud_file_ids": True, - }, - ) - - def test_should_allow_legacy_object_path_with_trusted_server_flag(self, config): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) + def test_should_allow_legacy_object_path_when_the_call_sets_the_flag(self, config): bucket, encoded = config._parse_gcs_uri( "gs://my-bucket/private/object.txt", litellm_params={ "gcs_bucket_name": "my-bucket", - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) assert bucket == "my-bucket" assert encoded == urllib.parse.quote("private/object.txt", safe="") - def test_should_reject_user_supplied_legacy_flag_snapshot(self, config): - with pytest.raises(ValueError, match="LiteLLM-managed"): - config._parse_gcs_uri( - "gs://my-bucket/private/object.txt", - litellm_params={ - "gcs_bucket_name": "my-bucket", - "_litellm_internal_model_credentials": {"allow_legacy_cloud_file_ids": True}, - }, - ) - def test_should_keep_configured_prefix_for_legacy_object_path(self, config): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) bucket, encoded = config._parse_gcs_uri( "gs://my-bucket/team-a/private/object.txt", litellm_params={ "gcs_bucket_name": "my-bucket/team-a", - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) @@ -119,13 +97,12 @@ class TestParseGcsUri: assert encoded == urllib.parse.quote("team-a/private/object.txt", safe="") def test_should_reject_legacy_object_outside_configured_prefix(self, config): - trusted_credentials = MappingProxyType({"allow_legacy_cloud_file_ids": True}) with pytest.raises(ValueError, match="configured storage prefix"): config._parse_gcs_uri( "gs://my-bucket/team-b/private/object.txt", litellm_params={ "gcs_bucket_name": "my-bucket/team-a", - "_litellm_internal_model_credentials": trusted_credentials, + "allow_legacy_cloud_file_ids": True, }, ) diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 928c73d4b04..8651d522ac8 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -5311,6 +5311,124 @@ def test_get_end_user_id_from_request_body_backwards_compatibility(): assert get_end_user_id_from_request_body(request_body) == expected_end_user_id +_BUCKET_ALLOWLIST: Final = {"allowed_client_side_s3_buckets": ["team-batch-bucket"]} +_ENDPOINT_ALLOWLIST: Final = {"allowed_client_side_s3_endpoint_urls": ["https://s3.us-west-2.amazonaws.com"]} + + +def _batches_body_is_safe(body: dict[str, object], general_settings: dict[str, object]) -> bool: + return is_request_body_safe( + request_body=body, + general_settings=general_settings, + llm_router=None, + model="", + route="/v1/batches", + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("s3_bucket_name", "team-batch-bucket"), + ("s3_output_bucket_name", "team-batch-bucket"), + ("allow_legacy_cloud_file_ids", True), + ("s3_endpoint_url", "https://s3.us-west-2.amazonaws.com"), + ], +) +def test_storage_setting_in_request_body_is_rejected_without_an_allowlist(field: str, value: object) -> None: + with pytest.raises(ValueError, match="Rejected Request") as error: + _batches_body_is_safe({field: value}, {}) + + assert field in str(error.value) + + +@pytest.mark.parametrize("field", ["s3_bucket_name", "s3_output_bucket_name"]) +def test_bucket_on_the_proxy_allowlist_is_accepted(field: str) -> None: + assert _batches_body_is_safe({field: "team-batch-bucket"}, _BUCKET_ALLOWLIST) is True + + +@pytest.mark.parametrize("field", ["s3_bucket_name", "s3_output_bucket_name"]) +def test_bucket_off_the_proxy_allowlist_is_rejected_and_names_the_setting(field: str) -> None: + with pytest.raises(ValueError, match=field) as error: + _batches_body_is_safe({field: "someone-elses-bucket"}, _BUCKET_ALLOWLIST) + + assert "general_settings.allowed_client_side_s3_buckets" in str(error.value) + + +@pytest.mark.parametrize( + "general_settings,accepted_body,rejected_field", + [ + ( + {**_BUCKET_ALLOWLIST, "allowed_client_side_s3_endpoint_urls": None}, + {"s3_bucket_name": "team-batch-bucket"}, + "s3_endpoint_url", + ), + ( + {**_ENDPOINT_ALLOWLIST, "allowed_client_side_s3_buckets": None}, + {"s3_endpoint_url": "https://s3.us-west-2.amazonaws.com"}, + "s3_bucket_name", + ), + ], +) +def test_allowlist_left_null_in_config_is_treated_as_empty( + general_settings: dict[str, object], accepted_body: dict[str, object], rejected_field: str +) -> None: + assert _batches_body_is_safe(accepted_body, general_settings) is True + with pytest.raises(ValueError, match=f"{rejected_field} is not allowed"): + _batches_body_is_safe({rejected_field: "team-batch-bucket"}, general_settings) + + +def test_endpoint_on_the_proxy_allowlist_is_accepted() -> None: + assert _batches_body_is_safe({"s3_endpoint_url": "https://s3.us-west-2.amazonaws.com"}, _ENDPOINT_ALLOWLIST) is True + + +def test_endpoint_off_the_proxy_allowlist_is_rejected_and_names_the_setting() -> None: + with pytest.raises(ValueError, match="s3_endpoint_url") as error: + _batches_body_is_safe({"s3_endpoint_url": "https://attacker.example"}, _ENDPOINT_ALLOWLIST) + + assert "general_settings.allowed_client_side_s3_endpoint_urls" in str(error.value) + + +def test_bucket_allowlist_admits_neither_an_endpoint_nor_the_legacy_file_flag() -> None: + with pytest.raises(ValueError, match="s3_endpoint_url"): + _batches_body_is_safe( + {"s3_bucket_name": "team-batch-bucket", "s3_endpoint_url": "team-batch-bucket"}, + _BUCKET_ALLOWLIST, + ) + with pytest.raises(ValueError, match="allow_legacy_cloud_file_ids"): + _batches_body_is_safe( + {"s3_bucket_name": "team-batch-bucket", "allow_legacy_cloud_file_ids": True}, + _BUCKET_ALLOWLIST, + ) + + +def test_endpoint_allowlist_does_not_admit_a_bucket() -> None: + with pytest.raises(ValueError, match="s3_bucket_name"): + _batches_body_is_safe({"s3_bucket_name": "https://s3.us-west-2.amazonaws.com"}, _ENDPOINT_ALLOWLIST) + + +def test_bucket_nested_in_extra_body_is_checked_against_the_allowlist() -> None: + assert _batches_body_is_safe({"extra_body": {"s3_bucket_name": "team-batch-bucket"}}, _BUCKET_ALLOWLIST) is True + with pytest.raises(ValueError, match="s3_bucket_name"): + _batches_body_is_safe({"extra_body": {"s3_bucket_name": "someone-elses-bucket"}}, _BUCKET_ALLOWLIST) + + +def test_proxy_wide_client_side_credentials_still_admit_any_bucket() -> None: + assert ( + _batches_body_is_safe( + {"s3_bucket_name": "any-bucket", "allow_legacy_cloud_file_ids": True}, + {"allow_client_side_credentials": True}, + ) + is True + ) + + +def test_other_blocked_fields_do_not_point_at_the_storage_allowlist() -> None: + with pytest.raises(ValueError, match="api_base") as error: + _batches_body_is_safe({"api_base": "https://attacker.example"}, _BUCKET_ALLOWLIST) + + assert "allowed_client_side" not in str(error.value) + + def _request_with_content_length(content_length: int) -> Request: return Request( { diff --git a/tests/unit/proxy/batches_endpoints/test_endpoints.py b/tests/unit/proxy/batches_endpoints/test_endpoints.py index 24f7f128b0a..b5da6f31eb4 100644 --- a/tests/unit/proxy/batches_endpoints/test_endpoints.py +++ b/tests/unit/proxy/batches_endpoints/test_endpoints.py @@ -1723,11 +1723,6 @@ async def test_retrieve__unified_batch_id_routes_to_router(retrieve_harness): assert retrieve_harness.router_aretrieve.call_count == 1 retrieve_harness.litellm_aretrieve.assert_not_called() - # Credentials are resolved for the deployment behind the unified id so the batch's - # output file can be read for cost accounting. This id resolves to nothing here, and - # the retrieve must still serve the batch rather than fail on the lookup. - retrieve_harness.creds_resolver.assert_called_once_with("gpt-4o-mini", form="with_provider") - # router receives the (still-encoded) batch id verbatim - this layer does # not decode it for the unified path. assert retrieve_harness.router_kwargs() == {"batch_id": "batch-unified-blob"} diff --git a/tests/unit/proxy/common_utils/test_check_batch_cost.py b/tests/unit/proxy/common_utils/test_check_batch_cost.py index eee5d96511b..09c76b2dd65 100644 --- a/tests/unit/proxy/common_utils/test_check_batch_cost.py +++ b/tests/unit/proxy/common_utils/test_check_batch_cost.py @@ -362,17 +362,19 @@ class TestCheckBatchCost: assert update_data["status"] == "complete" @pytest.mark.asyncio - async def test_output_fetch_passes_deployment_credentials_as_trusted_snapshot( - self, check_batch_cost_instance, mock_prisma_client, mock_llm_router + async def test_output_fetch_reads_from_the_deployment_bucket( + self, check_batch_cost_instance, mock_prisma_client, mock_llm_router, monkeypatch ): - """Bedrock resolves the output bucket ONLY from the immutable snapshot kwarg. + """Without the deployment's s3_bucket_name the cost poller cannot read a Bedrock + batch's output file, and every completed Bedrock batch stays unbilled.""" + import httpx - Spreading the credentials as plain kwargs is not enough: get_litellm_params drops - s3_bucket_name, so without _litellm_internal_model_credentials the cost poller - cannot read the output file and every completed Bedrock batch stays unbilled. - """ - from types import MappingProxyType - from unittest.mock import patch + import litellm + + monkeypatch.delenv("AWS_S3_BUCKET_NAME", raising=False) + monkeypatch.delenv("AWS_S3_OUTPUT_BUCKET_NAME", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1) mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() @@ -386,7 +388,7 @@ class TestCheckBatchCost: mock_response = MagicMock() mock_response.status = "completed" - mock_response.output_file_id = "file-output-123" + mock_response.output_file_id = "s3://configured-batch-bucket/litellm-batch-outputs/job-1/input.jsonl.out" mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}' mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response) @@ -395,6 +397,8 @@ class TestCheckBatchCost: "custom_llm_provider": "bedrock", "s3_bucket_name": "configured-batch-bucket", "aws_region_name": "us-east-1", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", } ) @@ -404,12 +408,10 @@ class TestCheckBatchCost: mock_deployment.model_info.model_dump.return_value = {} mock_llm_router.registry.return_value.deployment = MagicMock(return_value=mock_deployment) - mock_file_content = MagicMock() - mock_file_content.content = b'{"recordId":"req-1"}' - decoded_id = "llm_model_id,model-123;llm_batch_id,batch-456;" with ( + respx.mock(assert_all_called=False) as s3, patch( "litellm.proxy.openai_files_endpoints.common_utils._is_base64_encoded_unified_file_id", side_effect=[decoded_id, None], @@ -422,11 +424,6 @@ class TestCheckBatchCost: "litellm.proxy.openai_files_endpoints.common_utils.get_batch_id_from_unified_batch_id", return_value="batch-456", ), - patch( - "litellm.files.main.afile_content", - new_callable=AsyncMock, - return_value=mock_file_content, - ) as mock_afile_content, patch( "litellm.batches.batch_utils.get_file_content_as_dictionary", return_value=[{"recordId": "req-1"}], @@ -447,17 +444,15 @@ class TestCheckBatchCost: mock_logging_obj = MagicMock() mock_logging_obj.async_success_handler = AsyncMock() mock_logging_cls.return_value = mock_logging_obj + s3.get(url__regex=r"^https://s3\.us-east-1\.amazonaws\.com/").mock( + return_value=httpx.Response(200, content=b'{"recordId":"req-1"}') + ) await check_batch_cost_instance.check_batch_cost() - mock_afile_content.assert_awaited() - passed_kwargs = mock_afile_content.await_args[1] - snapshot = passed_kwargs.get("_litellm_internal_model_credentials") - assert snapshot is not None, "cost poller must pass the trusted credential snapshot" - assert isinstance(snapshot, MappingProxyType), ( - "snapshot must be a MappingProxyType; a plain dict is rejected by get_configured_s3_bucket_name" - ) - assert snapshot["s3_bucket_name"] == "configured-batch-bucket" + assert [str(call.request.url) for call in s3.calls] == [ + "https://s3.us-east-1.amazonaws.com/configured-batch-bucket/litellm-batch-outputs/job-1/input.jsonl.out" + ] @pytest.mark.asyncio async def test_poller_prices_with_deployment_registered_batch_rates( diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 238f1fa6495..b3323de3de5 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -396,74 +396,6 @@ async def test_a_caller_that_handed_off_accounting_still_leaves_the_marker_alone assert data["status"] == "complete" -# =========================================================================== # -# add_internal_model_credentials - the snapshot that lets a completed -# batch's output file be read, and therefore its cost be recorded -# =========================================================================== # - - -def test_add_internal_model_credentials_attaches_an_immutable_snapshot(): - """Cost accounting for a completed batch reads its output file, and Bedrock resolves - that bucket only from this snapshot. It must be immutable so nothing downstream can - redirect the bucket that managed file ids are validated against.""" - from litellm.proxy.openai_files_endpoints.common_utils import ( - add_internal_model_credentials, - ) - - router = route_entry_points_to_stubs(MagicMock()) - router.get_deployment_credentials_with_provider = MagicMock( - return_value={"s3_bucket_name": "configured-bucket", "aws_region_name": "us-east-1"} - ) - data = {"batch_id": "unified-batch-id"} - - add_internal_model_credentials(data=data, llm_router=router, model_id="deployment-1") - - snapshot = data["_litellm_internal_model_credentials"] - assert snapshot["s3_bucket_name"] == "configured-bucket" - assert isinstance(snapshot, MappingProxyType) - with pytest.raises(TypeError): - snapshot["s3_bucket_name"] = "attacker-bucket" - router.get_deployment_credentials_with_provider.assert_called_once_with(model_id="deployment-1") - - -@pytest.mark.parametrize( - "model_id, credentials", - [(None, {"s3_bucket_name": "b"}), ("deployment-1", None)], - ids=["no-model-id", "deployment-has-no-credentials"], -) -def test_add_internal_model_credentials_is_a_noop_without_a_resolvable_deployment(model_id, credentials): - """An unroutable batch must be left alone rather than given an empty snapshot, which - would look like a configured bucket of nothing.""" - from litellm.proxy.openai_files_endpoints.common_utils import ( - add_internal_model_credentials, - ) - - router = route_entry_points_to_stubs(MagicMock()) - router.get_deployment_credentials_with_provider = MagicMock(return_value=credentials) - data = {"batch_id": "unified-batch-id"} - - add_internal_model_credentials(data=data, llm_router=router, model_id=model_id) - - assert "_litellm_internal_model_credentials" not in data - - -def test_add_internal_model_credentials_survives_a_failing_deployment_lookup(): - """The snapshot only enables cost accounting, so a batch whose deployment no longer - resolves, which happens when a model group is removed while batches are in flight, - must still be retrievable rather than failing the request on the lookup.""" - from litellm.proxy.openai_files_endpoints.common_utils import ( - add_internal_model_credentials, - ) - - router = route_entry_points_to_stubs(MagicMock()) - router.get_deployment_credentials_with_provider = MagicMock(side_effect=KeyError("deployment-gone")) - data = {"batch_id": "unified-batch-id"} - - add_internal_model_credentials(data=data, llm_router=router, model_id="deployment-gone") - - assert data == {"batch_id": "unified-batch-id"} - - from openai.types.batch import BatchRequestCounts from litellm.proxy.openai_files_endpoints.common_utils import ( diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index ff63e37bcd8..27a65a91c41 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -4128,28 +4128,28 @@ def test_require_managed_files_rejects_raw_provider_file_id( mock_call.assert_not_called() -def test_get_file_content_model_routed_attaches_trusted_model_credentials(monkeypatch): - """A managed batch output id routes by model, and that branch must build the snapshot. - - The managed-files pre-call hook sets data["model"] for any id carrying - llm_output_file_id, so batch output retrieval always takes the model-routed branch - and never reaches managed_files_obj.afile_content. Bedrock resolves its output - bucket only from _litellm_internal_model_credentials, so without the snapshot every - Bedrock batch output retrieval fails with "S3 bucket_name is required". - """ +def test_get_file_content_model_routed_reads_from_the_deployment_bucket( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A managed batch output id routes by model, so Bedrock batch output retrieval only + works when the deployment's s3_bucket_name reaches the SDK. Without it every + retrieval fails with "S3 bucket_name is required".""" import base64 - from types import MappingProxyType import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles from litellm.types.utils import SpecialEnums + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router = Router( model_list=[ { "model_name": "anthropic.batch.claude-4.5-haiku", "litellm_params": { "model": "bedrock/anthropic.claude-haiku-4-5-20251001-v1:0", + "aws_access_key_id": "AKIAEXAMPLE", + "aws_secret_access_key": "secret", "aws_region_name": "us-east-1", "s3_bucket_name": "configured-batch-bucket", }, @@ -4173,27 +4173,11 @@ def test_get_file_content_model_routed_attaches_trusted_model_credentials(monkey monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router) - # One frozen snapshot per call rather than one dict merged across calls, so a second - # invocation is visible instead of silently overwriting the first. - calls: list[MappingProxyType] = [] - - async def _mock_router_afile_content(**kwargs): - calls.append(MappingProxyType(dict(kwargs))) - return HttpxBinaryResponseContent( - response=httpx.Response( - status_code=200, - content=b'{"recordId":"req-1"}', - headers={"content-type": "application/octet-stream"}, - ) - ) - - monkeypatch.setattr(router, "afile_content", _mock_router_afile_content) - unified_id = SpecialEnums.LITELLM_MANAGED_FILE_COMPLETE_STR.value.format( "application/jsonl", "unified-output-id", "anthropic.batch.claude-4.5-haiku", - "llm_output_file_id,s3://configured-batch-bucket/out/batch.jsonl", + "s3://configured-batch-bucket/litellm-batch-outputs/job/output.jsonl", "bedrock-batch-deployment-id", ) encoded_id = base64.urlsafe_b64encode(unified_id.encode()).decode().rstrip("=") @@ -4204,21 +4188,22 @@ def test_get_file_content_model_routed_attaches_trusted_model_credentials(monkey user_id="test-user", ) try: - response = client.get( - f"/v1/files/{encoded_id}/content", - headers={"Authorization": "Bearer test-key", "custom-llm-provider": "bedrock"}, - ) + with respx.mock(assert_all_called=False) as s3: + s3_read: Final = s3.get(url__regex=r"https://.*amazonaws\.com/.*").mock( + return_value=httpx.Response(200, content=b'{"recordId":"req-1"}') + ) + response = client.get( + f"/v1/files/{encoded_id}/content", + headers={"Authorization": "Bearer test-key", "custom-llm-provider": "bedrock"}, + ) finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) assert response.status_code == 200, response.text - assert len(calls) == 1, f"expected exactly one routed retrieval, got {len(calls)}" - snapshot = calls[0].get("_litellm_internal_model_credentials") - assert snapshot is not None, "model-routed branch must attach the trusted credential snapshot" - assert isinstance( - snapshot, MappingProxyType - ), "snapshot must be a MappingProxyType; a plain dict is rejected by get_configured_s3_bucket_name" - assert snapshot["s3_bucket_name"] == "configured-batch-bucket" + assert response.content == b'{"recordId":"req-1"}' + assert [call.request.url.path for call in s3_read.calls] == [ + "/configured-batch-bucket/litellm-batch-outputs/job/output.jsonl" + ] def _unified_managed_file_id() -> str: @@ -5471,16 +5456,11 @@ def test_create_file_path_traversal_filename_rejected_before_forwarding(monkeypa assert forwarded_calls == [] -def test_list_files_target_model_names_passes_trusted_bedrock_credentials( - mocker: MockerFixture, monkeypatch -): - """ - GET /v1/files?target_model_names= must hand the deployment's - immutable credential snapshot to litellm.afile_list, since Bedrock resolves - the S3 bucket to list from that snapshot rather than from request params. - """ - from types import MappingProxyType - +def test_list_files_target_model_names_passes_the_deployment_bucket( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + """GET /v1/files?target_model_names= lists the deployment's bucket, + so its s3_bucket_name must reach litellm.afile_list.""" import litellm.proxy.proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -5532,9 +5512,7 @@ def test_list_files_target_model_names_passes_trusted_bedrock_credentials( assert response.status_code == 200, response.text assert captured_kwargs["custom_llm_provider"] == "bedrock" assert captured_kwargs["purpose"] == "batch" - trusted_credentials = captured_kwargs["_litellm_internal_model_credentials"] - assert isinstance(trusted_credentials, MappingProxyType) - assert trusted_credentials["s3_bucket_name"] == "my-bucket" + assert captured_kwargs["s3_bucket_name"] == "my-bucket" proxy_logging_obj.post_call_failure_hook.assert_not_called() diff --git a/tests/unit/proxy/test_model_based_routing_files_batches.py b/tests/unit/proxy/test_model_based_routing_files_batches.py index f9bc07212a2..f586dadffc4 100644 --- a/tests/unit/proxy/test_model_based_routing_files_batches.py +++ b/tests/unit/proxy/test_model_based_routing_files_batches.py @@ -6,8 +6,6 @@ Tests the model-based routing ID encoding/decoding used by the batch and file proxy endpoints. """ -from types import MappingProxyType - from litellm.proxy.openai_files_endpoints.common_utils import ( decode_model_from_file_id, encode_file_id_with_model, @@ -54,40 +52,24 @@ class TestEncodeFileIdWithModel: class TestPrepareDataWithCredentials: - def test_preserves_trusted_internal_credentials_snapshot(self): - data = {"file_id": "file-abc"} + def test_deployment_bucket_settings_override_the_request_and_reach_the_sdk_as_kwargs(self): + data = {"file_id": "file-abc", "s3_bucket_name": "request-bucket"} credentials = { "custom_llm_provider": "bedrock", - "s3_bucket_name": "safe-bucket", + "s3_bucket_name": "deployment-bucket", + "s3_output_bucket_name": "deployment-output-bucket", + "allow_legacy_cloud_file_ids": True, } - prepare_data_with_credentials( - data=data, - credentials=credentials, - include_internal_credentials=True, - ) + prepare_data_with_credentials(data=data, credentials=credentials, file_id="decoded-file-id") - assert data["s3_bucket_name"] == "safe-bucket" - assert "custom_llm_provider" not in data - assert isinstance( - data["_litellm_internal_model_credentials"], type(MappingProxyType({})) - ) - assert ( - data["_litellm_internal_model_credentials"]["s3_bucket_name"] - == "safe-bucket" - ) - - def test_does_not_add_internal_credentials_by_default(self): - data = {"file_id": "file-abc"} - credentials = { - "custom_llm_provider": "bedrock", - "s3_bucket_name": "safe-bucket", + assert data == { + "file_id": "decoded-file-id", + "s3_bucket_name": "deployment-bucket", + "s3_output_bucket_name": "deployment-output-bucket", + "allow_legacy_cloud_file_ids": True, } - prepare_data_with_credentials(data=data, credentials=credentials) - - assert "_litellm_internal_model_credentials" not in data - class TestRoundTrip: """Tests for encode -> decode round-trip integrity.""" diff --git a/tests/unit/test_azure_ad_token_credential_resolution.py b/tests/unit/test_azure_ad_token_credential_resolution.py index f6f47b3f4c4..b1abc1d3c1e 100644 --- a/tests/unit/test_azure_ad_token_credential_resolution.py +++ b/tests/unit/test_azure_ad_token_credential_resolution.py @@ -149,11 +149,11 @@ class TestRouterCredentialResolutionS3OutputBucket: """Same strict-dump trap as azure_ad_token (#30235), for Bedrock batch file retrieval (#26335). Bedrock batch outputs land in a per-model ``s3_output_bucket_name`` when it differs from the input bucket. The - file-content retrieval path validates a file id against the buckets in the - trusted credential snapshot, and that snapshot is built by round-tripping - the deployment's ``litellm_params`` through ``CredentialLiteLLMParams``. If - the field is undeclared it is dropped, so the output bucket never reaches - retrieval and output-bucket file ids are rejected as foreign.""" + file-content retrieval path validates a file id against the buckets the + proxy passes along, and those are built by round-tripping the deployment's + ``litellm_params`` through ``CredentialLiteLLMParams``. If the field is + undeclared it is dropped, so the output bucket never reaches retrieval and + output-bucket file ids are rejected as foreign.""" def test_credentials_preserve_s3_output_bucket_name(self): from litellm import Router @@ -185,6 +185,33 @@ class TestRouterCredentialResolutionS3OutputBucket: ) assert credentials.get("s3_bucket_name") == "in-bucket" + def test_credentials_preserve_allow_legacy_cloud_file_ids(self): + """An admin who sets allow_legacy_cloud_file_ids on a deployment expects + legacy (pre-managed-prefix) file ids in its bucket to stay readable + through the proxy's model-routed file endpoints.""" + from litellm import Router + + deployment_id = "bedrock-batch-legacy-ids-fixed-uuid" + router = Router( + model_list=[ + { + "model_name": "bedrock-batch-legacy-ids", + "litellm_params": { + "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + "s3_bucket_name": "in-bucket", + "allow_legacy_cloud_file_ids": True, + "aws_region_name": "us-west-2", + }, + "model_info": {"id": deployment_id}, + } + ] + ) + + credentials = router.get_deployment_credentials_with_provider(model_id=deployment_id) + + assert credentials is not None + assert credentials.get("allow_legacy_cloud_file_ids") is True + def test_credentials_without_output_bucket_unaffected(self): """A deployment that configures only the input bucket keeps it and does not gain a phantom output bucket in the resolved credentials.""" diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index f9dae53bd03..d319252506d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30791,6 +30791,16 @@ export interface components { * @description If True, lets keys address Responses API ids that this proxy did not issue (raw provider ids, or ids issued before response-id encryption was configured). Such an id carries no owner, so no ownership check can run on it; ids this proxy did issue keep full ownership enforcement. Off by default, in which case an unrecognized response id is rejected with 403 */ allow_unmanaged_response_ids?: boolean | null; + /** + * Allowed Client Side S3 Buckets + * @description S3 bucket names a client may send as `s3_bucket_name` or `s3_output_bucket_name` in a request body, e.g. on /v1/batches. Any other bucket is rejected unless client-side credentials are allowed + */ + allowed_client_side_s3_buckets?: string[] | null; + /** + * Allowed Client Side S3 Endpoint Urls + * @description Exact `s3_endpoint_url` values a client may send in a request body. Any other value is rejected unless client-side credentials are allowed + */ + allowed_client_side_s3_endpoint_urls?: string[] | null; /** * Allowed File Extensions * @description the only file extensions (e.g. ['.jsonl', '.pdf', '.txt']) accepted on /v1/files uploads, for any purpose, matched case-insensitively against the uploaded filename. Files with any other extension, or none, are rejected. An empty list rejects every upload. Unset means no allowlist is applied @@ -36286,6 +36296,8 @@ export interface components { * @default false */ allow_client_keepalive_override: boolean | null; + /** Allow Legacy Cloud File Ids */ + allow_legacy_cloud_file_ids?: boolean | null; /** Annotation Cost Per Page */ annotation_cost_per_page?: number | null; /** Annotation Cost Per Page Batches */ @@ -52381,6 +52393,8 @@ export interface components { * @default false */ allow_client_keepalive_override: boolean | null; + /** Allow Legacy Cloud File Ids */ + allow_legacy_cloud_file_ids?: boolean | null; /** Annotation Cost Per Page */ annotation_cost_per_page?: number | null; /** Annotation Cost Per Page Batches */