From 39bd5fb8b7923f94712bac1bd9f1f6a93adb50ad Mon Sep 17 00:00:00 2001 From: Sujith Date: Tue, 14 Jul 2026 15:18:16 +0530 Subject: [PATCH 01/62] fix(main): forward store and prompt_cache_key params on chat completions (#33184) --- litellm/main.py | 8 ++++ litellm/utils.py | 2 + tests/test_litellm/test_main.py | 84 +++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) diff --git a/litellm/main.py b/litellm/main.py index 7d457d9cdd1..81b02082abd 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -435,6 +435,8 @@ async def acompletion( verbosity: Optional[Literal["low", "medium", "high"]] = None, safety_identifier: Optional[str] = None, service_tier: Optional[str] = None, + store: Optional[bool] = None, + prompt_cache_key: Optional[str] = None, # set api_base, api_version, api_key base_url: Optional[str] = None, api_version: Optional[str] = None, @@ -585,6 +587,8 @@ async def acompletion( "verbosity": verbosity, "safety_identifier": safety_identifier, "service_tier": service_tier, + "store": store, + "prompt_cache_key": prompt_cache_key, "extra_headers": extra_headers, "acompletion": True, # assuming this is a required parameter "thinking": thinking, @@ -4828,6 +4832,8 @@ def completion( # type: ignore extra_headers: Optional[dict] = None, safety_identifier: Optional[str] = None, service_tier: Optional[str] = None, + store: Optional[bool] = None, + prompt_cache_key: Optional[str] = None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, function_call: Optional[str] = None, @@ -5249,6 +5255,8 @@ def completion( # type: ignore ), "safety_identifier": safety_identifier, "service_tier": service_tier, + "store": store, + "prompt_cache_key": prompt_cache_key, "allowed_openai_params": kwargs.get("allowed_openai_params"), "base_model": base_model, } diff --git a/litellm/utils.py b/litellm/utils.py index 18b89ee0d13..b16ecdf88be 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3791,6 +3791,8 @@ def get_optional_params( thinking: Optional[AnthropicThinkingParam] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, safety_identifier: Optional[str] = None, + store: Optional[bool] = None, + prompt_cache_key: Optional[str] = None, base_model: Optional[str] = None, **kwargs, ): diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py index 28cf4fa0744..0624b590df7 100644 --- a/tests/test_litellm/test_main.py +++ b/tests/test_litellm/test_main.py @@ -2081,3 +2081,87 @@ def test_stream_chunk_builder_text_completion_combines_text_and_usage(): assert response.usage.prompt_tokens > 0 assert response.usage.completion_tokens > 0 assert response.usage.total_tokens == response.usage.prompt_tokens + response.usage.completion_tokens + + +def test_completion_forwards_store_and_prompt_cache_key_to_openai(): + """ + Regression test for https://github.com/BerriAI/litellm/issues/33184 + + store and prompt_cache_key are documented OpenAI chat completion params that + were accepted as supported but silently dropped before the provider request + was built, because they were not named parameters of completion() and + get_optional_params() the way safety_identifier is. + """ + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key") + + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + try: + litellm.completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello"}], + store=False, + prompt_cache_key="test-cache-key", + client=client, + ) + except Exception as e: + print(e) + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + assert request_body["store"] is False + assert request_body["prompt_cache_key"] == "test-cache-key" + + +@pytest.mark.asyncio +async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai(): + """ + Async variant of the store/prompt_cache_key forwarding regression test for + https://github.com/BerriAI/litellm/issues/33184 + """ + from openai import AsyncOpenAI + + client = AsyncOpenAI(api_key="fake-api-key") + + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + try: + await litellm.acompletion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello"}], + store=False, + prompt_cache_key="test-cache-key", + client=client, + ) + except Exception as e: + print(e) + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + assert request_body["store"] is False + assert request_body["prompt_cache_key"] == "test-cache-key" + + +def test_completion_omits_store_and_prompt_cache_key_when_not_passed(): + """ + When store and prompt_cache_key are not passed, they must not appear in the + outbound request body (guards against always forwarding None defaults). + """ + from openai import OpenAI + + client = OpenAI(api_key="fake-api-key") + + with patch.object(client.chat.completions.with_raw_response, "create") as mock_client: + try: + litellm.completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello"}], + client=client, + ) + except Exception as e: + print(e) + + mock_client.assert_called_once() + request_body = mock_client.call_args.kwargs + assert "store" not in request_body + assert "prompt_cache_key" not in request_body From 4eaa70440a247cee2767bd16e7dc830da559105a Mon Sep 17 00:00:00 2001 From: Sujith Date: Tue, 14 Jul 2026 15:49:46 +0530 Subject: [PATCH 02/62] fix(main): use PEP 604 unions for new store and prompt_cache_key params --- litellm/main.py | 8 ++++---- litellm/utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 81b02082abd..3b13e04bbe3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -435,8 +435,8 @@ async def acompletion( verbosity: Optional[Literal["low", "medium", "high"]] = None, safety_identifier: Optional[str] = None, service_tier: Optional[str] = None, - store: Optional[bool] = None, - prompt_cache_key: Optional[str] = None, + store: bool | None = None, + prompt_cache_key: str | None = None, # set api_base, api_version, api_key base_url: Optional[str] = None, api_version: Optional[str] = None, @@ -4832,8 +4832,8 @@ def completion( # type: ignore extra_headers: Optional[dict] = None, safety_identifier: Optional[str] = None, service_tier: Optional[str] = None, - store: Optional[bool] = None, - prompt_cache_key: Optional[str] = None, + store: bool | None = None, + prompt_cache_key: str | None = None, # soon to be deprecated params by OpenAI functions: Optional[List] = None, function_call: Optional[str] = None, diff --git a/litellm/utils.py b/litellm/utils.py index b16ecdf88be..bc5b2447761 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3791,8 +3791,8 @@ def get_optional_params( thinking: Optional[AnthropicThinkingParam] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, safety_identifier: Optional[str] = None, - store: Optional[bool] = None, - prompt_cache_key: Optional[str] = None, + store: bool | None = None, + prompt_cache_key: str | None = None, base_model: Optional[str] = None, **kwargs, ): From 371fa670d6f79dfd579945e2d357f5b978d9af21 Mon Sep 17 00:00:00 2001 From: shivam Date: Fri, 17 Jul 2026 19:33:28 +0000 Subject: [PATCH 03/62] fix(proxy): forward Bedrock event-stream content-type on unbuffered passthrough Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 14 +++-- .../proxy/test_common_request_processing.py | 55 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c7c9397d850..24831a64410 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1766,6 +1766,7 @@ class ProxyBaseLLMRequestProcessing: return StreamingResponse( content=generator, # type: ignore[arg-type] status_code=status.HTTP_200_OK, + media_type=self._passthrough_event_stream_media_type(), headers=custom_headers, ) else: @@ -2216,10 +2217,15 @@ class ProxyBaseLLMRequestProcessing: def _passthrough_event_stream_media_type(self) -> Optional[str]: """ - Content-type for a buffered passthrough event-stream response, resolved - from the provider handler so the proxy stays provider-agnostic. Mirrors - the upstream content-type the non-streaming path forwards, since the - buffered streaming generator carries no headers of its own. + Content-type for a passthrough event-stream response, resolved from the + provider handler so the proxy stays provider-agnostic. Mirrors the + upstream content-type the non-streaming path forwards, since the + streaming generator carries no headers of its own. Used for both the + buffered (guardrail-rewritten) and the unbuffered relay paths so + clients that enforce the event-stream content-type (e.g. Claude Code on + Bedrock invoke-with-response-stream) see the correct header instead of + Starlette's application/octet-stream default. Returns None for providers + with no event-stream media type, leaving the response default unchanged. """ from litellm.llms.pass_through.guardrail_translation.handler import ( LlmPassthroughRouteHandler, diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index ebfbb46053d..f1a3745e85b 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4137,6 +4137,61 @@ class TestAllmPassthroughStreamingProviderGate: assert streamed == chunks mock_handler.assert_not_awaited() + @pytest.mark.asyncio + async def test_bedrock_invoke_stream_sets_event_stream_content_type(self, monkeypatch): + """ + Regression for LIT-4561. The unbuffered Bedrock event-stream relay + (invoke-with-response-stream, no post-call guardrail rewriting) must set + content-type: application/vnd.amazon.eventstream instead of leaving it to + Starlette's application/octet-stream default, which trips Claude Code's + content-type guard added in 2.1.208 + """ + processing_obj = self._build_processing_obj( + "bedrock", "model/us.anthropic.claude-sonnet-4-20250514-v1:0/invoke-with-response-stream" + ) + chunks = [b"raw-1", b"raw-2"] + + with patch.object( + ProxyBaseLLMRequestProcessing, + "_has_post_call_guardrails", + return_value=False, + ), patch.object( + ProxyBaseLLMRequestProcessing, + "_has_post_call_guardrails_for_passthrough", + return_value=False, + ): + result = await self._run(processing_obj, monkeypatch, chunks) + + assert isinstance(result, StreamingResponse) + assert result.media_type == "application/vnd.amazon.eventstream" + assert result.headers["content-type"] == "application/vnd.amazon.eventstream" + streamed = [chunk async for chunk in result.body_iterator] + assert streamed == chunks + + @pytest.mark.asyncio + async def test_non_bedrock_stream_keeps_default_content_type(self, monkeypatch): + """ + A provider with no registered event-stream media type must not have one + forced onto its unbuffered stream, so the response default is unchanged + """ + processing_obj = self._build_processing_obj("anthropic") + chunks = [b"chunk-1", b"chunk-2"] + + with patch.object( + ProxyBaseLLMRequestProcessing, + "_has_post_call_guardrails", + return_value=False, + ), patch.object( + ProxyBaseLLMRequestProcessing, + "_has_post_call_guardrails_for_passthrough", + return_value=False, + ): + result = await self._run(processing_obj, monkeypatch, chunks) + + assert isinstance(result, StreamingResponse) + assert result.media_type is None + assert result.headers.get("content-type") != "application/vnd.amazon.eventstream" + class TestResponseCostHeaderForTypedDictResponses: """ From db061d6e3118806dd59340e900af482b15450326 Mon Sep 17 00:00:00 2001 From: ayaangazali Date: Thu, 23 Jul 2026 16:25:13 -0700 Subject: [PATCH 04/62] fix(azure_ai): strip non-OpenAI-spec message fields before request --- litellm/llms/azure_ai/chat/transformation.py | 21 +++++- .../chat/test_azure_ai_transformation.py | 72 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 5540d79f667..683c05cfbaa 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -11,6 +11,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( _audio_or_image_in_message_content, convert_content_list_to_str, + filter_value_from_dict, ) from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj @@ -28,6 +29,13 @@ class AzureFoundryErrorStrings(str, enum.Enum): SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'" +NON_OPENAI_SPEC_MESSAGE_FIELDS = ( + "thinking_blocks", + "provider_specific_fields", + "cache_control", +) + + class AzureAIStudioConfig(OpenAIConfig): def get_supported_openai_params(self, model: str) -> list: model_supports_tool_choice = True # azure ai supports this by default @@ -167,10 +175,19 @@ class AzureAIStudioConfig(OpenAIConfig): ) -> list: """ - Azure AI Studio doesn't support content as a list. This handles: - 1. Transforms list content to a string. - 2. If message contains an image or audio, send as is (user-intended) + 1. Strips message fields that are not part of the OpenAI chat-completions + schema (thinking_blocks, provider_specific_fields, cache_control). + Azure AI Foundry backends set additionalProperties=false and reject + these with "Extra inputs are not permitted", which breaks multi-turn + Anthropic-format clients that echo thinking blocks back as history. + 2. Transforms list content to a string. + 3. If message contains an image or audio, send as is (user-intended) """ for message in messages: + message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped in place + for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: + filter_value_from_dict(message_dict, field) + # Do nothing if the message contains an image or audio if _audio_or_image_in_message_content(message): continue diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index 2e75039139c..80c4355b560 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -262,3 +262,75 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name(): assert "copilot_mcp_server_name" not in tool assert result["tools"][0]["type"] == "function" assert result["tools"][1]["function"]["name"] == "read_file" + + +def _find_key_anywhere(obj, key: str) -> bool: + if isinstance(obj, dict): + if key in obj: + return True + return any(_find_key_anywhere(v, key) for v in obj.values()) + if isinstance(obj, list): + return any(_find_key_anywhere(item, key) for item in obj) + return False + + +def test_azure_ai_strips_non_openai_spec_message_fields(): + """ + Regression for https://github.com/BerriAI/litellm/issues/33961. + + Azure AI Foundry backends set additionalProperties=false, so any message + field outside the OpenAI chat-completions schema causes a 400 "Extra inputs + are not permitted". Anthropic-format clients (e.g. Claude Code) echo prior + assistant turns back as history carrying thinking_blocks, a nested thought + signature at tool_calls[].function.provider_specific_fields, and Anthropic + cache_control annotations. transform_request must strip all of these before + the request reaches the upstream. + """ + config = AzureAIStudioConfig() + + messages = [ + {"role": "user", "content": "Read a file."}, + { + "role": "assistant", + "content": "I can help.", + "thinking_blocks": [ + { + "type": "thinking", + "thinking": "The user wants me to read a file.", + "signature": "", + "cache_control": {"type": "ephemeral"}, + } + ], + "provider_specific_fields": {"thought_signature": "sig-top"}, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "read_file", + "arguments": "{}", + "provider_specific_fields": {"thought_signature": "sig-nested"}, + }, + } + ], + }, + {"role": "user", "content": "go ahead"}, + ] + + request = config.transform_request( + model="fw-glm-5.2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + transformed_messages = request["messages"] + + assert not _find_key_anywhere(transformed_messages, "thinking_blocks") + assert not _find_key_anywhere(transformed_messages, "provider_specific_fields") + assert not _find_key_anywhere(transformed_messages, "cache_control") + + assistant_message = transformed_messages[1] + assert assistant_message["content"] == "I can help." + assert assistant_message["tool_calls"][0]["function"]["name"] == "read_file" From 95bc890fcfc157247072752e4005686dd9413a54 Mon Sep 17 00:00:00 2001 From: ayaangazali Date: Thu, 30 Jul 2026 11:42:03 -0700 Subject: [PATCH 05/62] fix(azure_ai): strip non-spec message fields on a copy, not the caller's messages --- litellm/llms/azure_ai/chat/transformation.py | 7 ++- .../chat/test_azure_ai_transformation.py | 50 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 683c05cfbaa..067c89214ab 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,3 +1,4 @@ +import copy import enum import re from typing import Any, Final, cast @@ -182,9 +183,13 @@ class AzureAIStudioConfig(OpenAIConfig): Anthropic-format clients that echo thinking blocks back as history. 2. Transforms list content to a string. 3. If message contains an image or audio, send as is (user-intended) + + Operates on a deep copy so the caller's messages keep their thinking blocks + and provider metadata, which a fallback to another provider still needs. """ + messages = copy.deepcopy(messages) for message in messages: - message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped in place + message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: filter_value_from_dict(message_dict, field) diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index 80c4355b560..beb7e9dfab0 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -334,3 +334,53 @@ def test_azure_ai_strips_non_openai_spec_message_fields(): assistant_message = transformed_messages[1] assert assistant_message["content"] == "I can help." assert assistant_message["tool_calls"][0]["function"]["name"] == "read_file" + + +def test_azure_ai_stripping_does_not_mutate_caller_messages(): + """ + The stripping must not touch the caller's messages. LiteLLM reuses the same + message objects when falling back to another provider, so stripping in place + would hand the fallback a conversation history with its thinking blocks and + provider metadata already destroyed. + """ + config = AzureAIStudioConfig() + + messages = [ + {"role": "user", "content": "Read a file."}, + { + "role": "assistant", + "content": "I can help.", + "thinking_blocks": [ + {"type": "thinking", "thinking": "Reading the file.", "signature": "sig"} + ], + "provider_specific_fields": {"thought_signature": "sig-top"}, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "read_file", + "arguments": "{}", + "provider_specific_fields": {"thought_signature": "sig-nested"}, + }, + } + ], + }, + ] + + request = config.transform_request( + model="fw-glm-5.2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert not _find_key_anywhere(request["messages"], "thinking_blocks") + + original_assistant = messages[1] + assert original_assistant["thinking_blocks"][0]["thinking"] == "Reading the file." + assert original_assistant["provider_specific_fields"] == {"thought_signature": "sig-top"} + assert original_assistant["tool_calls"][0]["function"]["provider_specific_fields"] == { + "thought_signature": "sig-nested" + } From d0c1d2be8a82723d458eaf133159ad8576bd9a8b Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 7 Aug 2026 18:08:33 -0700 Subject: [PATCH 06/62] feat(key_management): let any authenticated user resolve a raw key via /key/info Possession of the raw sk- key already lets the holder call /key/info with the key itself as the bearer token, so resolving raw key -> key info for any authenticated caller discloses nothing new. Lookups by hashed token remain restricted to admins, the key's owner, and teammates --- .../key_management_endpoints.py | 7 + .../test_key_management_endpoints.py | 146 ++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index e4def45892b..99631377541 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -6351,6 +6351,12 @@ async def _can_user_query_key_info( ) -> bool: """ Helper to check if the user has access to the key's info + + Any authenticated caller who presents the raw key value (i.e. a preimage of the + stored token hash) is allowed: possession of the raw key already grants the + ability to call /key/info with that key as the bearer token, so resolving + raw key -> key info discloses nothing new. Lookups by hashed token remain + restricted to admins, the key's owner, and the key's teammates. """ if ( ( @@ -6359,6 +6365,7 @@ async def _can_user_query_key_info( ) or user_api_key_dict.api_key == key or key_info.user_id == user_api_key_dict.user_id + or (key is not None and hash_token(token=key) == key_info.token) or await TeamMemberPermissionChecks.user_belongs_to_keys_team( user_api_key_dict=user_api_key_dict, existing_key_row=key_info, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index cf9aa477112..fc349f8448d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -15323,3 +15323,149 @@ async def test_rotate_master_key_rotates_sso_identity_assertions( prisma_client=mock_prisma_client, new_master_key="sk-new-master-key", ) + + +@pytest.mark.asyncio +async def test_can_user_query_key_info_raw_key_possession_allows_any_user(): + """ + Any authenticated user who presents the raw sk- key value can query that + key's info: possessing the raw key already lets them call /key/info with + the key itself as the bearer token, so this discloses nothing new. + """ + from litellm.proxy._types import hash_token + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _can_user_query_key_info, + ) + + raw_key = "sk-raw-key-owned-by-someone-else" + key_info = LiteLLM_VerificationToken( + token=hash_token(raw_key), + user_id="key-owner", + team_id=None, + key_alias="prod-batch-alias", + ) + caller = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="unrelated-user", + api_key="hashed-caller-token", + ) + + assert ( + await _can_user_query_key_info( + user_api_key_dict=caller, + key=raw_key, + key_info=key_info, + ) + is True + ) + + +@pytest.mark.asyncio +async def test_can_user_query_key_info_hashed_token_still_forbidden(): + """ + Querying by hashed token (e.g. copied from spend logs) must stay + restricted to admins, the key's owner, and teammates. + """ + from litellm.proxy._types import hash_token + from litellm.proxy.management_endpoints.key_management_endpoints import ( + _can_user_query_key_info, + ) + + raw_key = "sk-raw-key-owned-by-someone-else" + key_info = LiteLLM_VerificationToken( + token=hash_token(raw_key), + user_id="key-owner", + team_id=None, + key_alias="prod-batch-alias", + ) + caller = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="unrelated-user", + api_key="hashed-caller-token", + ) + + assert ( + await _can_user_query_key_info( + user_api_key_dict=caller, + key=hash_token(raw_key), + key_info=key_info, + ) + is False + ) + + +@pytest.mark.asyncio +async def test_info_key_fn_resolves_alias_from_raw_key_for_any_user(monkeypatch): + """ + End-to-end through /key/info: a non-admin user unrelated to the key can + resolve raw sk- key -> key_alias. + """ + from litellm.proxy._types import hash_token + from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn + + raw_key = "sk-raw-key-owned-by-someone-else" + key_row = LiteLLM_VerificationToken( + token=hash_token(raw_key), + user_id="key-owner", + team_id=None, + key_alias="prod-batch-alias", + ) + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", AsyncMock()) + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=key_row + ) + + caller = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="unrelated-user", + api_key="hashed-caller-token", + ) + + result = await info_key_fn(key=raw_key, user_api_key_dict=caller) + + assert result["info"]["key_alias"] == "prod-batch-alias" + assert "token" not in result["info"] + + find_unique_kwargs = ( + mock_prisma_client.db.litellm_verificationtoken.find_unique.call_args.kwargs + ) + assert find_unique_kwargs["where"] == {"token": hash_token(raw_key)} + + +@pytest.mark.asyncio +async def test_info_key_fn_hashed_lookup_still_403_for_unrelated_user(monkeypatch): + """ + End-to-end through /key/info: the same unrelated user querying by hashed + token still gets a 403. + """ + from litellm.proxy._types import hash_token + from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn + + raw_key = "sk-raw-key-owned-by-someone-else" + key_row = LiteLLM_VerificationToken( + token=hash_token(raw_key), + user_id="key-owner", + team_id=None, + key_alias="prod-batch-alias", + ) + + mock_prisma_client = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", AsyncMock()) + mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=key_row + ) + + caller = UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="unrelated-user", + api_key="hashed-caller-token", + ) + + with pytest.raises(ProxyException) as exc_info: + await info_key_fn(key=hash_token(raw_key), user_api_key_dict=caller) + + assert int(exc_info.value.code) == 403 From a14b2ab960d093fe1198caac649f8f9884b9a773 Mon Sep 17 00:00:00 2001 From: Shivam Rawat Date: Fri, 7 Aug 2026 18:47:27 -0700 Subject: [PATCH 07/62] fix(router): forward target_model_names on file uploads to litellm_proxy deployments Uploads for the Batch API through a deployment that points at a second LiteLLM proxy arrived downstream as bare multipart requests with no model or target_model_names, so the second proxy could not route them and fell back to files_settings or the wrong endpoint shape. The router now injects target_model_names into extra_body when the deployment provider is litellm_proxy, and litellm_proxy is registered as an OpenAI-compatible files/batches provider so the downstream call uses the deployment api_base and api_key over the OpenAI wire format. Resolves https://github.com/BerriAI/litellm/issues/36176 --- litellm/files/main.py | 9 +- litellm/router.py | 9 ++ litellm/types/utils.py | 1 + tests/test_litellm/test_router.py | 135 ++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 3 deletions(-) diff --git a/litellm/files/main.py b/litellm/files/main.py index 34421d13761..cf8826895aa 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -24,12 +24,15 @@ FileCreateProvider = Literal[ "vertex_ai", "bedrock", "hosted_vllm", + "litellm_proxy", "manus", "anthropic", ] -FileRetrieveProvider = Literal["openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "manus", "anthropic"] -FileDeleteProvider = Literal["openai", "azure", "gemini", "manus", "anthropic"] -FileListProvider = Literal["openai", "azure", "manus", "anthropic"] +FileRetrieveProvider = Literal[ + "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic" +] +FileDeleteProvider = Literal["openai", "azure", "gemini", "litellm_proxy", "manus", "anthropic"] +FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic"] import litellm from litellm import get_secret_str from litellm.files.streaming import FileContentStreamingResponse diff --git a/litellm/router.py b/litellm/router.py index 39d5080a33d..b35506554d1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -21,6 +21,7 @@ import traceback from collections import defaultdict from collections.abc import AsyncGenerator, Callable, Generator, Mapping, Sequence from functools import lru_cache +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeVar, Union, cast import anyio @@ -192,6 +193,7 @@ from litellm.types.utils import ( CustomPricingLiteLLMParams, GenericBudgetConfigType, LiteLLMBatch, + LlmProviders, ModelInfo, ModelResponseStream, StandardLoggingPayload, @@ -4919,6 +4921,13 @@ class Router: ) kwargs_copy["file"] = file + if custom_llm_provider == LlmProviders.LITELLM_PROXY.value: + kwargs_copy["extra_body"] = MappingProxyType( + { + **(kwargs_copy.get("extra_body") or MappingProxyType({})), + "target_model_names": stripped_model, + } + ) if ( "gcs_bucket_name" in data ): # TODO: Remove this once we have a better way to handle GCS bucket name: Problem is that we need to pass the gcs_bucket_name to the router for the create_file call but it doesn't show up there diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4f824262867..20225d7dc49 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3706,6 +3706,7 @@ LlmProvidersSet: Final = {provider.value for provider in LlmProviders} OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS: set[str] = { LlmProviders.OPENAI.value, LlmProviders.HOSTED_VLLM.value, + LlmProviders.LITELLM_PROXY.value, } ListBatchesSupportedProvider = Literal["openai", "azure", "hosted_vllm", "vertex_ai"] diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 8f35597768f..92085a009af 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -522,6 +522,141 @@ async def test_async_router_acreate_file_uses_deployment_custom_llm_provider(): assert mock_acreate_file.call_args.kwargs["custom_llm_provider"] == "azure" +@pytest.mark.asyncio +async def test_async_router_acreate_file_forwards_target_model_names_to_litellm_proxy(): + """ + A deployment pointing at a second LiteLLM proxy (litellm_proxy provider) must forward + target_model_names downstream so the second proxy can route the upload to the right + deployment. Regression test for https://github.com/BerriAI/litellm/issues/36176 + """ + import json + from io import BytesIO + from unittest.mock import MagicMock, patch + + jsonl_file = BytesIO( + json.dumps({"body": {"model": "chained-batch", "messages": [{"role": "user", "content": "hi"}]}}).encode( + "utf-8" + ) + ) + jsonl_file.name = "test.jsonl" + + router = litellm.Router( + model_list=[ + { + "model_name": "chained-batch", + "litellm_params": { + "model": "litellm_proxy/gpt-4.1-batch", + "api_base": "http://localhost:4001/v1", + "api_key": "sk-proxy-b", + }, + }, + ], + ) + + with patch("litellm.acreate_file", return_value=MagicMock()) as mock_acreate_file: + await router.acreate_file( + model="chained-batch", + purpose="batch", + file=jsonl_file, + ) + + assert mock_acreate_file.call_count == 1 + call_kwargs = mock_acreate_file.call_args.kwargs + assert call_kwargs["custom_llm_provider"] == "litellm_proxy" + assert call_kwargs["extra_body"] == {"target_model_names": "gpt-4.1-batch"} + uploaded_line = json.loads(call_kwargs["file"].read().decode("utf-8").split("\n")[0]) + assert uploaded_line["body"]["model"] == "gpt-4.1-batch" + + +@pytest.mark.asyncio +async def test_async_router_acreate_file_does_not_inject_target_model_names_for_other_providers(): + """ + target_model_names is a LiteLLM proxy routing hint; it must not leak into uploads + sent to non-litellm_proxy providers. + """ + from unittest.mock import MagicMock, patch + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4.1-batch", + "litellm_params": {"model": "gpt-4.1"}, + }, + ], + ) + + with patch("litellm.acreate_file", return_value=MagicMock()) as mock_acreate_file: + await router.acreate_file( + model="gpt-4.1-batch", + purpose="batch", + file=MagicMock(), + ) + + assert mock_acreate_file.call_count == 1 + assert mock_acreate_file.call_args.kwargs.get("extra_body") is None + + +@pytest.mark.asyncio +async def test_async_router_acreate_file_litellm_proxy_sends_target_model_names_in_multipart_form(): + """ + End-to-end through litellm.acreate_file and the OpenAI SDK: the multipart form that + reaches the second proxy must carry target_model_names as a form field, since the + downstream /v1/files endpoint reads it via Form(). Would raise BadRequestError + (unsupported provider) before litellm_proxy was supported for files. + """ + import json + from io import BytesIO + + import httpx + import respx + + jsonl_file = BytesIO( + json.dumps({"body": {"model": "chained-batch", "messages": [{"role": "user", "content": "hi"}]}}).encode( + "utf-8" + ) + ) + jsonl_file.name = "test.jsonl" + + router = litellm.Router( + model_list=[ + { + "model_name": "chained-batch", + "litellm_params": { + "model": "litellm_proxy/gpt-4.1-batch", + "api_base": "http://localhost:4001/v1", + "api_key": "sk-proxy-b", + }, + }, + ], + ) + + file_object_json = { + "id": "file-abc123", + "object": "file", + "bytes": 100, + "created_at": 1700000000, + "filename": "test.jsonl", + "purpose": "batch", + "status": "processed", + } + + with respx.mock(assert_all_called=True) as respx_mock: + create_route = respx_mock.post("http://localhost:4001/v1/files").mock( + return_value=httpx.Response(200, json=file_object_json) + ) + response = await router.acreate_file( + model="chained-batch", + purpose="batch", + file=jsonl_file, + ) + + assert response.id == "file-abc123" + request_body = create_route.calls.last.request.content + assert b'name="target_model_names"' in request_body + assert b"gpt-4.1-batch" in request_body + assert b'name="purpose"' in request_body + + @pytest.mark.asyncio async def test_async_router_afile_content_uses_deployment_custom_llm_provider(): """ From 3c96030488914eeea8a67dfc4d50944b79cc4c3b Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 8 Aug 2026 02:20:35 +0000 Subject: [PATCH 08/62] fix(advisor): resolve the advisor sub-call through the proxy router Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/interceptors/advisor.py | 79 +++++++- .../messages/test_advisor_orchestration.py | 190 ++++++++++++++++++ 2 files changed, 264 insertions(+), 5 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py index dfae7b4f4cf..805f5625ad9 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py @@ -16,7 +16,7 @@ How it works: import uuid from collections.abc import AsyncIterator -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import litellm import litellm.constants as _c @@ -28,6 +28,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +if TYPE_CHECKING: + from litellm.router import Router + ADVISOR_MAX_USES: Final[int] = _c.ADVISOR_MAX_USES ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = _c.ADVISOR_NATIVE_PROVIDERS ADVISOR_TOOL_DESCRIPTION: Final[str] = _c.ADVISOR_TOOL_DESCRIPTION @@ -138,13 +141,10 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): # --- Advisor sub-call (always non-streaming, no tools) --- try: - advisor_response: AnthropicMessagesResponse = await _call_messages_handler( + advisor_response: AnthropicMessagesResponse = await _call_advisor( model=advisor_model, messages=advisor_messages, - tools=None, - stream=False, max_tokens=max_tokens, - custom_llm_provider=None, # let litellm resolve from model name metadata={ **metadata_base, "advisor_sub_call": True, @@ -357,6 +357,75 @@ def _inject_max_uses_error( ] +def _resolve_advisor_router(advisor_model: str) -> "Router | None": + """Return the proxy router when it can resolve ``advisor_model``. + + The advisor sub-call must honor the proxy's ``model_list`` (and its + fallbacks / credentials) exactly like a direct call to that model group + would. Without this, provider resolution falls back to the bare model + name, which for a ``claude-*`` advisor model means the public Anthropic + API, bypassing the configured deployment entirely. + + Returns ``None`` for SDK callers (no proxy router) and for advisor models + the router doesn't know about, so those keep resolving through + ``litellm.anthropic_messages()`` provider inference. + """ + try: + from litellm.proxy.proxy_server import llm_router + except (ImportError, ModuleNotFoundError): + return None + if llm_router is None: + return None + if llm_router.get_model_list(model_name=advisor_model): + return llm_router + if llm_router.model_group_alias and advisor_model in llm_router.model_group_alias: + return llm_router + if llm_router.pattern_router.route(advisor_model) is not None: + return llm_router + return None + + +async def _call_advisor( + *, + model: str, + messages: list[dict], + max_tokens: int, + metadata: dict, + api_key: str | None, + api_base: str | None, +) -> AnthropicMessagesResponse: + """Run the advisor sub-call, through the proxy router when it applies. + + A caller-supplied ``api_key`` / ``api_base`` override is an explicit + request to bypass the configured deployment, so it keeps the direct + SDK-level path. + """ + router: Final = None if (api_key or api_base) else _resolve_advisor_router(model) + response: Final = ( + await router.aanthropic_messages( + model=model, + messages=messages, + tools=None, + stream=False, + max_tokens=max_tokens, + metadata=metadata, + ) + if router is not None + else await _call_messages_handler( + model=model, + messages=messages, + tools=None, + stream=False, + max_tokens=max_tokens, + custom_llm_provider=None, + metadata=metadata, + api_key=api_key, + api_base=api_base, + ) + ) + return cast(AnthropicMessagesResponse, response) # cast-ok: both /messages entry points are untyped + + async def _call_messages_handler( model: str, messages: list[dict], diff --git a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py index 3d35e93167f..bcb1843f408 100644 --- a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py @@ -1041,3 +1041,193 @@ async def test_executor_failure_is_not_tagged(): ) assert is_advisor_orchestration_failure(exc_info.value) is False + + +# --------------------------------------------------------------------------- +# 15. The advisor sub-call resolves through the proxy router when the advisor +# model is configured in model_list, instead of dialing the public +# Anthropic API (regression for LIT-5307). +# --------------------------------------------------------------------------- + + +def _router_with_advisor_deployment(recorder, advisor_model="claude-opus-4-8"): + """Build a Router whose only deployment is the advisor model on Foundry. + + The recorder replaces ``litellm.anthropic_messages`` before construction + because Router binds it at init time, so the returned Router exercises the + real deployment-resolution path and records what it dispatched. + """ + import litellm + from litellm.router import Router + + with patch("litellm.anthropic_messages", new=recorder): + return Router( + model_list=[ + { + "model_name": advisor_model, + "litellm_params": { + "model": f"azure_ai/{advisor_model}", + "api_base": "http://127.0.0.1:1/foundry", + "api_key": "fake-foundry-key", + }, + } + ], + num_retries=0, + ) + + +@pytest.mark.asyncio +async def test_advisor_sub_call_routes_through_proxy_router(): + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + router_calls = [] + + async def recorder(**kwargs): + router_calls.append(kwargs) + return _make_text_response("Use trial division.", model="claude-opus-4-8") + + router = _router_with_advisor_deployment(recorder) + + call_count = 0 + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _make_advisor_tool_use_response() + return _make_text_response("Final answer.") + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ), + patch.object(proxy_server, "llm_router", router), + ): + h = AdvisorOrchestrationHandler() + result = await h.handle( + model="executor-model", + messages=MESSAGES, + tools=[{**ADVISOR_TOOL, "model": "claude-opus-4-8"}], + stream=False, + max_tokens=512, + custom_llm_provider="azure_ai", + ) + + assert call_count == 2 + assert len(router_calls) == 1 + assert router_calls[0]["model"] == "azure_ai/claude-opus-4-8" + assert router_calls[0]["api_base"] == "http://127.0.0.1:1/foundry" + assert router_calls[0]["api_key"] == "fake-foundry-key" + assert "Final answer." in result["content"][0]["text"] + + +@pytest.mark.asyncio +async def test_advisor_sub_call_bypasses_router_for_unconfigured_model(): + """An advisor model the router doesn't know about keeps the SDK-level path.""" + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + router_calls = [] + + async def recorder(**kwargs): + router_calls.append(kwargs) + return _make_text_response("should not be used") + + router = _router_with_advisor_deployment(recorder, advisor_model="some-other-model") + + call_count = 0 + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _make_advisor_tool_use_response() + if tools is None: + return _make_text_response("Advice.", model="claude-opus-4-8") + return _make_text_response("Final answer.") + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ), + patch.object(proxy_server, "llm_router", router), + ): + h = AdvisorOrchestrationHandler() + await h.handle( + model="executor-model", + messages=MESSAGES, + tools=[{**ADVISOR_TOOL, "model": "claude-opus-4-8"}], + stream=False, + max_tokens=512, + custom_llm_provider="azure_ai", + ) + + assert router_calls == [] + assert call_count == 3 + + +@pytest.mark.asyncio +async def test_advisor_sub_call_client_override_bypasses_router(): + """A caller-supplied api_key/api_base override must not be re-routed.""" + import litellm + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + router_calls = [] + + async def recorder(**kwargs): + router_calls.append(kwargs) + return _make_text_response("should not be used") + + router = _router_with_advisor_deployment(recorder) + + sub_calls = [] + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + sub_calls.append({"model": model, "tools": tools, **kwargs}) + if len(sub_calls) == 1: + return _make_advisor_tool_use_response() + if tools is None: + return _make_text_response("Advice.", model="claude-opus-4-8") + return _make_text_response("Final answer.") + + advisor_tool = { + **ADVISOR_TOOL, + "model": "claude-opus-4-8", + "api_key": "client-key", + "api_base": "https://client.example.com", + } + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ), + patch.object(proxy_server, "llm_router", router), + patch.dict(proxy_server.general_settings, {"allow_client_side_credentials": True}), + patch.object(litellm, "user_url_validation", False), + ): + h = AdvisorOrchestrationHandler() + await h.handle( + model="executor-model", + messages=MESSAGES, + tools=[advisor_tool], + stream=False, + max_tokens=512, + custom_llm_provider="azure_ai", + ) + + assert router_calls == [] + advisor_sub_calls = [c for c in sub_calls if c["tools"] is None] + assert len(advisor_sub_calls) == 1 + assert advisor_sub_calls[0]["api_key"] == "client-key" + assert advisor_sub_calls[0]["api_base"] == "https://client.example.com" From 45317d58641c0ebcb1a84868d79fbcc19fca7b02 Mon Sep 17 00:00:00 2001 From: shivam Date: Sat, 8 Aug 2026 02:48:08 +0000 Subject: [PATCH 09/62] refactor(advisor): resolve advisor router once per request Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../messages/interceptors/advisor.py | 83 +++++++------------ 1 file changed, 30 insertions(+), 53 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py index 805f5625ad9..adf96db61c2 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py @@ -16,7 +16,7 @@ How it works: import uuid from collections.abc import AsyncIterator -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final import litellm import litellm.constants as _c @@ -100,6 +100,14 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): parent_request_id: Final[str] = str(kwargs.pop("litellm_call_id", None) or uuid.uuid4()) metadata_base: Final[dict] = dict(kwargs.pop("metadata", None) or {}) + advisor_metadata: Final = { + **metadata_base, + "advisor_sub_call": True, + "parent_request_id": parent_request_id, + } + advisor_router: Final = ( + None if (advisor_api_key or advisor_api_base) else _resolve_advisor_router(advisor_model) + ) iteration = 0 while True: @@ -141,17 +149,27 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): # --- Advisor sub-call (always non-streaming, no tools) --- try: - advisor_response: AnthropicMessagesResponse = await _call_advisor( - model=advisor_model, - messages=advisor_messages, - max_tokens=max_tokens, - metadata={ - **metadata_base, - "advisor_sub_call": True, - "parent_request_id": parent_request_id, - }, - api_key=advisor_api_key, - api_base=advisor_api_base, + advisor_response: AnthropicMessagesResponse = ( + await advisor_router.aanthropic_messages( + model=advisor_model, + messages=advisor_messages, + tools=None, + stream=False, + max_tokens=max_tokens, + metadata=advisor_metadata, + ) + if advisor_router is not None + else await _call_messages_handler( + model=advisor_model, + messages=advisor_messages, + tools=None, + stream=False, + max_tokens=max_tokens, + custom_llm_provider=None, + metadata=advisor_metadata, + api_key=advisor_api_key, + api_base=advisor_api_base, + ) ) except Exception as advisor_sub_call_exception: mark_advisor_orchestration_failure(advisor_sub_call_exception) @@ -385,47 +403,6 @@ def _resolve_advisor_router(advisor_model: str) -> "Router | None": return None -async def _call_advisor( - *, - model: str, - messages: list[dict], - max_tokens: int, - metadata: dict, - api_key: str | None, - api_base: str | None, -) -> AnthropicMessagesResponse: - """Run the advisor sub-call, through the proxy router when it applies. - - A caller-supplied ``api_key`` / ``api_base`` override is an explicit - request to bypass the configured deployment, so it keeps the direct - SDK-level path. - """ - router: Final = None if (api_key or api_base) else _resolve_advisor_router(model) - response: Final = ( - await router.aanthropic_messages( - model=model, - messages=messages, - tools=None, - stream=False, - max_tokens=max_tokens, - metadata=metadata, - ) - if router is not None - else await _call_messages_handler( - model=model, - messages=messages, - tools=None, - stream=False, - max_tokens=max_tokens, - custom_llm_provider=None, - metadata=metadata, - api_key=api_key, - api_base=api_base, - ) - ) - return cast(AnthropicMessagesResponse, response) # cast-ok: both /messages entry points are untyped - - async def _call_messages_handler( model: str, messages: list[dict], From f86dc8f54e9bbee403c185e02196610e6ef76bd2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:09:17 -0700 Subject: [PATCH 10/62] test(proxy): assert non-Bedrock passthrough stream emits no content-type header --- tests/test_litellm/proxy/test_common_request_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 7b762457bc8..e567d19dac7 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4564,7 +4564,7 @@ class TestAllmPassthroughStreamingProviderGate: assert isinstance(result, StreamingResponse) assert result.media_type is None - assert result.headers.get("content-type") != "application/vnd.amazon.eventstream" + assert "content-type" not in result.headers class TestResponseCostHeaderForTypedDictResponses: From d5a4c145778a66df0df5a6a627cae3e40e162645 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:21:28 -0700 Subject: [PATCH 11/62] docs(proxy): pre-fix passthrough streams omitted content-type, not octet-stream --- litellm/proxy/common_request_processing.py | 5 +++-- tests/test_litellm/proxy/test_common_request_processing.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 165e58c56f7..376d7785e5a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2502,8 +2502,9 @@ class ProxyBaseLLMRequestProcessing: buffered (guardrail-rewritten) and the unbuffered relay paths so clients that enforce the event-stream content-type (e.g. Claude Code on Bedrock invoke-with-response-stream) see the correct header instead of - Starlette's application/octet-stream default. Returns None for providers - with no event-stream media type, leaving the response default unchanged. + no content-type at all, which they fall back to reading as + application/octet-stream. Returns None for providers with no + event-stream media type, leaving the response headers unchanged. """ from litellm.llms.pass_through.guardrail_translation.handler import ( LlmPassthroughRouteHandler, diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index e567d19dac7..ddb7b0d510f 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -4516,9 +4516,9 @@ class TestAllmPassthroughStreamingProviderGate: """ Regression for LIT-4561. The unbuffered Bedrock event-stream relay (invoke-with-response-stream, no post-call guardrail rewriting) must set - content-type: application/vnd.amazon.eventstream instead of leaving it to - Starlette's application/octet-stream default, which trips Claude Code's - content-type guard added in 2.1.208 + content-type: application/vnd.amazon.eventstream instead of emitting no + content-type header at all, which trips Claude Code's content-type guard + added in 2.1.208 """ processing_obj = self._build_processing_obj( "bedrock", "model/us.anthropic.claude-sonnet-4-20250514-v1:0/invoke-with-response-stream" From dc54b16d3c84c87835d365e193c6f029dbe6aa81 Mon Sep 17 00:00:00 2001 From: Bruno Felthes Date: Mon, 17 Aug 2026 21:43:24 -0300 Subject: [PATCH 12/62] fix(fireworks): skip accounts/ rewrite for FW-* Foundry deployment ids resolve_fireworks_resource_name prefixes bare names with accounts/fireworks/models/ (or routers/ for *-fast). Azure AI Foundry hosts Fireworks models under deployment ids like FW-Kimi-K3; rewriting those yields 404 DeploymentNotFound. Leave names that already start with FW- unchanged. Native Fireworks short names still get the accounts/ path. Co-authored-by: Cursor --- litellm/llms/fireworks_ai/common_utils.py | 5 +++++ .../llms/fireworks_ai/test_fireworks_ai_common_utils.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index e07e7a26f9e..522c955770c 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -33,6 +33,11 @@ def resolve_fireworks_resource_name(model: str) -> str: stripped: Final = model.removeprefix("fireworks_ai/") if stripped.startswith("accounts/") or "#" in stripped: return stripped + # Azure AI Foundry (and similar OpenAI-compat hosts) expose Fireworks + # deployments as ids like ``FW-Kimi-K3``. Rewriting those to + # ``accounts/fireworks/models/FW-…`` yields DeploymentNotFound. + if stripped.startswith("FW-"): + return stripped if stripped.startswith(("routers/", "models/")): return f"accounts/fireworks/{stripped}" if stripped.endswith("-fast"): diff --git a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py index 4af395baf41..b52c910d5a6 100644 --- a/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py +++ b/tests/test_litellm/llms/fireworks_ai/test_fireworks_ai_common_utils.py @@ -39,6 +39,9 @@ from litellm.llms.fireworks_ai.common_utils import resolve_fireworks_resource_na "glm-4p6#accounts/gitlab/deployments/2fb7764c", "glm-4p6#accounts/gitlab/deployments/2fb7764c", ), + ("FW-Kimi-K3", "FW-Kimi-K3"), + ("fireworks_ai/FW-Kimi-K3", "FW-Kimi-K3"), + ("FW-GLM-5.2-Fast", "FW-GLM-5.2-Fast"), ], ) def test_resolve_fireworks_resource_name(model, expected): From dd1838936596967d528737dc121bd53bb7147c27 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:04:27 -0700 Subject: [PATCH 13/62] fix(router): route Responses API input through the auto-router Auto-router strategy hook returned None whenever the request carried input instead of messages, so tagged /v1/responses requests (Codex CLI) never picked a tier and tag filtering left nothing to route to. Resolve input through the shared prompt-template helper before matching routes. --- .../auto_router/auto_router.py | 7 +- .../router_strategy/test_auto_router.py | 67 +++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 20c0ece46b6..86383a27880 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -128,13 +128,14 @@ class AutoRouter(CustomLogger): """ from semantic_router.routers import SemanticRouter + from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages from litellm.router_strategy.auto_router.litellm_encoder import ( LiteLLMRouterEncoder, ) from litellm.types.router import PreRoutingHookResponse - if messages is None: - # do nothing, return same inputs + resolved_messages: Final = resolve_structured_messages(messages=messages, request_kwargs=request_kwargs) + if resolved_messages is None: return None routelayer = self.routelayer @@ -153,7 +154,7 @@ class AutoRouter(CustomLogger): ) self.routelayer = routelayer - message_content: Final = self._extract_text_from_messages(messages) + message_content: Final = self._extract_text_from_messages(resolved_messages) route_name: Final = self._matched_route_name(routelayer, message_content) return PreRoutingHookResponse( diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index b73485e6019..10d133e7311 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -479,3 +479,70 @@ class TestAutoRouterEmbeddingInputCap: assert auto_router.routelayer is not None assert auto_router.routelayer.encoder.max_input_chars == 777 + + +class TestAutoRouterRoutesResponsesApiInput: + """Responses API requests carry the prompt in `input`, not `messages`, and still have to reach the route layer.""" + + @pytest.mark.asyncio + async def test_should_route_a_string_input_when_messages_is_none(self): + from semantic_router.schema import RouteChoice + + layer: Final = FixedRouteLayer(RouteChoice(name="code-model")) + auto_router: Final = _auto_router(layer) + + result: Final = await auto_router.async_pre_routing_hook( + model="my-auto-router", + request_kwargs={ + "input": "fix this stack trace", + "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}, + }, + messages=None, + ) + + assert result is not None + assert result.model == "code-model" + assert result.messages is None + assert layer.seen_text == "fix this stack trace" + + @pytest.mark.asyncio + async def test_should_route_a_list_input_with_instructions_when_messages_is_none(self): + from semantic_router.schema import RouteChoice + + layer: Final = FixedRouteLayer(RouteChoice(name="code-model")) + auto_router: Final = _auto_router(layer) + + result: Final = await auto_router.async_pre_routing_hook( + model="my-auto-router", + request_kwargs={ + "instructions": "You are a coding agent.", + "input": [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "fix this stack trace"}], + } + ], + "litellm_metadata": {"user_api_key_request_route": "/v1/responses"}, + }, + messages=None, + ) + + assert result is not None + assert result.model == "code-model" + assert layer.seen_text is not None + assert "fix this stack trace" in layer.seen_text + + @pytest.mark.asyncio + async def test_should_skip_routing_when_neither_messages_nor_input_is_present(self): + layer: Final = FixedRouteLayer(None) + auto_router: Final = _auto_router(layer) + + result: Final = await auto_router.async_pre_routing_hook( + model="my-auto-router", + request_kwargs={"litellm_metadata": {"user_api_key_request_route": "/v1/responses"}}, + messages=None, + ) + + assert result is None + assert layer.seen_text is None From 791b478d775edc107fe81fa757b20c1f9b9f5911 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:18:53 -0700 Subject: [PATCH 14/62] fix(router): only resolve structured input when messages is absent Keeps an explicit empty messages list on the pre-existing path (default model, provider validation error) instead of dropping the routing decision and surfacing a misleading tags 401. --- .../router_strategy/auto_router/auto_router.py | 6 +++++- .../router_strategy/test_auto_router.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/litellm/router_strategy/auto_router/auto_router.py b/litellm/router_strategy/auto_router/auto_router.py index 86383a27880..c77745a498d 100644 --- a/litellm/router_strategy/auto_router/auto_router.py +++ b/litellm/router_strategy/auto_router/auto_router.py @@ -134,7 +134,11 @@ class AutoRouter(CustomLogger): ) from litellm.types.router import PreRoutingHookResponse - resolved_messages: Final = resolve_structured_messages(messages=messages, request_kwargs=request_kwargs) + resolved_messages: Final = ( + messages + if messages is not None + else resolve_structured_messages(messages=None, request_kwargs=request_kwargs) + ) if resolved_messages is None: return None diff --git a/tests/test_litellm/router_strategy/test_auto_router.py b/tests/test_litellm/router_strategy/test_auto_router.py index 10d133e7311..c71a6b0e27f 100644 --- a/tests/test_litellm/router_strategy/test_auto_router.py +++ b/tests/test_litellm/router_strategy/test_auto_router.py @@ -546,3 +546,18 @@ class TestAutoRouterRoutesResponsesApiInput: assert result is None assert layer.seen_text is None + + @pytest.mark.asyncio + async def test_should_keep_routing_an_empty_messages_list_to_the_default_model(self): + layer: Final = FixedRouteLayer(None) + auto_router: Final = _auto_router(layer) + + result: Final = await auto_router.async_pre_routing_hook( + model="my-auto-router", + request_kwargs={"messages": [], "litellm_metadata": {"user_api_key_request_route": "/v1/chat/completions"}}, + messages=[], + ) + + assert result is not None + assert result.model == "fallback-model" + assert layer.seen_text == "" From 0c5fcde88386f144e218ab41661393de5282b211 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:08:28 -0700 Subject: [PATCH 15/62] Revert "feat(key_management): let any authenticated user resolve a raw key via /key/info" This reverts commit d0c1d2be8a82723d458eaf133159ad8576bd9a8b. --- .../key_management_endpoints.py | 7 - .../test_key_management_endpoints.py | 146 ------------------ 2 files changed, 153 deletions(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 99631377541..e4def45892b 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -6351,12 +6351,6 @@ async def _can_user_query_key_info( ) -> bool: """ Helper to check if the user has access to the key's info - - Any authenticated caller who presents the raw key value (i.e. a preimage of the - stored token hash) is allowed: possession of the raw key already grants the - ability to call /key/info with that key as the bearer token, so resolving - raw key -> key info discloses nothing new. Lookups by hashed token remain - restricted to admins, the key's owner, and the key's teammates. """ if ( ( @@ -6365,7 +6359,6 @@ async def _can_user_query_key_info( ) or user_api_key_dict.api_key == key or key_info.user_id == user_api_key_dict.user_id - or (key is not None and hash_token(token=key) == key_info.token) or await TeamMemberPermissionChecks.user_belongs_to_keys_team( user_api_key_dict=user_api_key_dict, existing_key_row=key_info, diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index fc349f8448d..cf9aa477112 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -15323,149 +15323,3 @@ async def test_rotate_master_key_rotates_sso_identity_assertions( prisma_client=mock_prisma_client, new_master_key="sk-new-master-key", ) - - -@pytest.mark.asyncio -async def test_can_user_query_key_info_raw_key_possession_allows_any_user(): - """ - Any authenticated user who presents the raw sk- key value can query that - key's info: possessing the raw key already lets them call /key/info with - the key itself as the bearer token, so this discloses nothing new. - """ - from litellm.proxy._types import hash_token - from litellm.proxy.management_endpoints.key_management_endpoints import ( - _can_user_query_key_info, - ) - - raw_key = "sk-raw-key-owned-by-someone-else" - key_info = LiteLLM_VerificationToken( - token=hash_token(raw_key), - user_id="key-owner", - team_id=None, - key_alias="prod-batch-alias", - ) - caller = UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, - user_id="unrelated-user", - api_key="hashed-caller-token", - ) - - assert ( - await _can_user_query_key_info( - user_api_key_dict=caller, - key=raw_key, - key_info=key_info, - ) - is True - ) - - -@pytest.mark.asyncio -async def test_can_user_query_key_info_hashed_token_still_forbidden(): - """ - Querying by hashed token (e.g. copied from spend logs) must stay - restricted to admins, the key's owner, and teammates. - """ - from litellm.proxy._types import hash_token - from litellm.proxy.management_endpoints.key_management_endpoints import ( - _can_user_query_key_info, - ) - - raw_key = "sk-raw-key-owned-by-someone-else" - key_info = LiteLLM_VerificationToken( - token=hash_token(raw_key), - user_id="key-owner", - team_id=None, - key_alias="prod-batch-alias", - ) - caller = UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, - user_id="unrelated-user", - api_key="hashed-caller-token", - ) - - assert ( - await _can_user_query_key_info( - user_api_key_dict=caller, - key=hash_token(raw_key), - key_info=key_info, - ) - is False - ) - - -@pytest.mark.asyncio -async def test_info_key_fn_resolves_alias_from_raw_key_for_any_user(monkeypatch): - """ - End-to-end through /key/info: a non-admin user unrelated to the key can - resolve raw sk- key -> key_alias. - """ - from litellm.proxy._types import hash_token - from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn - - raw_key = "sk-raw-key-owned-by-someone-else" - key_row = LiteLLM_VerificationToken( - token=hash_token(raw_key), - user_id="key-owner", - team_id=None, - key_alias="prod-batch-alias", - ) - - mock_prisma_client = AsyncMock() - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", AsyncMock()) - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( - return_value=key_row - ) - - caller = UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, - user_id="unrelated-user", - api_key="hashed-caller-token", - ) - - result = await info_key_fn(key=raw_key, user_api_key_dict=caller) - - assert result["info"]["key_alias"] == "prod-batch-alias" - assert "token" not in result["info"] - - find_unique_kwargs = ( - mock_prisma_client.db.litellm_verificationtoken.find_unique.call_args.kwargs - ) - assert find_unique_kwargs["where"] == {"token": hash_token(raw_key)} - - -@pytest.mark.asyncio -async def test_info_key_fn_hashed_lookup_still_403_for_unrelated_user(monkeypatch): - """ - End-to-end through /key/info: the same unrelated user querying by hashed - token still gets a 403. - """ - from litellm.proxy._types import hash_token - from litellm.proxy.management_endpoints.key_management_endpoints import info_key_fn - - raw_key = "sk-raw-key-owned-by-someone-else" - key_row = LiteLLM_VerificationToken( - token=hash_token(raw_key), - user_id="key-owner", - team_id=None, - key_alias="prod-batch-alias", - ) - - mock_prisma_client = AsyncMock() - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", AsyncMock()) - mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( - return_value=key_row - ) - - caller = UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, - user_id="unrelated-user", - api_key="hashed-caller-token", - ) - - with pytest.raises(ProxyException) as exc_info: - await info_key_fn(key=hash_token(raw_key), user_api_key_dict=caller) - - assert int(exc_info.value.code) == 403 From 00e1f25e9b9db6ebdc2ae22b6bf84621d97c0da6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 18 Aug 2026 13:12:27 -0700 Subject: [PATCH 16/62] refactor(ui): migrate the login, onboarding and search tool forms to react-hook-form and shadcn (#37334) * refactor(ui): migrate login, onboarding and search tool forms to react-hook-form and shadcn Moves four forms off antd Form and Tremor widgets onto react-hook-form plus the shadcn kit, and onto semantic colour tokens so the screens are dark-mode ready. antd Modal and Alert stay as the shells. The submit payload is unchanged in all four. Each unit is pinned by a characterization test that was proven green against the antd original before any production code changed, and the pre-existing test files pass unedited. Extracts the search tool payload builder, which was duplicated verbatim between the create and edit forms, into searchToolPayload.ts with unit tests, and adds a shared PasswordInput so the four reveal toggles antd and Tremor gave for free are preserved on one component. * refactor(ui): keep the search tool Test Connection button an implicit submit Tremor's Button renders no type attribute, so inside a form it defaults to submit. The Test Connection button therefore fires both its own onClick and the form's onFinish today, which creates the search tool as a side effect of testing the connection. shadcn's Button renders type="button", so the naive swap silently dropped that second path. Restores parity with an explicit type="submit" and pins it with a test, so the double submit is recorded rather than quietly changed. Fixing it belongs in its own change. Also prunes the two now-stale eslint suppression counts for the migrated search tool files, scoped to those keys only. * refactor(ui): announce the login button spinner the way antd did antd's Button renders its loading indicator as role="img" with aria-label "loading", so a screen reader announces the request in flight. The shadcn spinner is a bare svg, which drops that. Labels it on the login button to match, as already done on the onboarding submit button. * fix(ui): drop noValidate from the migrated login, onboarding and search-tool forms antd's Form renders no novalidate attribute and its required rules emit aria-required rather than the native required attribute, so nothing in these four forms was ever gated by native constraint validation. The only type="email" input is disabled and readOnly, which bars it from validation in every browser. Measured in jsdom and again in Chrome against a live proxy: form.checkValidity() is true with the fields empty, a native submit reaches react-hook-form, and zod blocks it with the same messages. Removing the attribute keeps the rendered form faithful to antd, and keeps a constraint added later behaving the way antd would have behaved instead of being silently suppressed. * fix(ui): accept a null api_key when seeding the search tool edit form The list endpoint declares api_key as str | None and search_tool_info as dict | None, and the masking helper returns non-string values untouched, so a tool stored without an API key comes back as "api_key": null. zod's optional() accepts undefined and rejects null, so the edit form for any such tool failed with "expected string, received null" and could never be submitted. antd carried no schema and forwarded whatever the server sent. nullish() restores that, and the payload still forwards the null rather than coercing it to an empty string. The new case seeds both null vectors and fails without this change. * chore(ui): drop the narration comments from the search tool forms These restate the state change or the JSX block directly below them, which the repo's comment policy rules out, and both files are rewritten by this change rather than merely touched. --- ui/litellm-dashboard/eslint-suppressions.json | 5 +- .../CreateSearchTools.integration.test.tsx | 151 +++++++ .../_components/CreateSearchTools.tsx | 368 ++++++++++-------- .../SearchTools.integration.test.tsx | 191 +++++++++ .../search-tools/_components/SearchTools.tsx | 169 ++++---- .../_components/searchToolPayload.test.ts | 64 +++ .../_components/searchToolPayload.ts | 33 ++ .../search-tools/_components/types.tsx | 4 +- .../app/login/LoginPage.integration.test.tsx | 192 +++++++++ .../src/app/login/LoginPage.tsx | 363 +++++++++-------- .../OnboardingFormBody.integration.test.tsx | 61 +++ .../src/app/onboarding/OnboardingFormBody.tsx | 131 ++++--- .../src/components/shared/PasswordInput.tsx | 39 ++ 13 files changed, 1300 insertions(+), 471 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/SearchTools.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/searchToolPayload.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/searchToolPayload.ts create mode 100644 ui/litellm-dashboard/src/app/login/LoginPage.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/onboarding/OnboardingFormBody.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/shared/PasswordInput.tsx diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 2436cb0e3b7..69c16f2022b 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1346,9 +1346,6 @@ }, "src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx": { "no-restricted-imports": { - "count": 2 - }, - "react-hooks/set-state-in-effect": { "count": 1 } }, @@ -1357,7 +1354,7 @@ "count": 2 }, "no-restricted-imports": { - "count": 2 + "count": 1 }, "react-hooks/static-components": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.integration.test.tsx new file mode 100644 index 00000000000..553ba0428b4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.integration.test.tsx @@ -0,0 +1,151 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import * as networking from "@/components/networking"; +import CreateSearchTool from "./CreateSearchTools"; + +vi.mock("@/components/networking", () => ({ + createSearchTool: vi.fn(), + fetchAvailableSearchProviders: vi.fn(), +})); + +vi.mock("@/lib/toast", () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})); + +vi.mock("./SearchConnectionTest", () => ({ + default: () =>
, +})); + +const providers = [ + { provider_name: "perplexity", ui_friendly_name: "Perplexity AI" }, + { provider_name: "tavily", ui_friendly_name: "Tavily Search" }, +]; + +const renderModal = () => { + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }); + return render( + + + , + ); +}; + +const pickProvider = async (user: ReturnType, label: string) => { + await user.click(screen.getAllByRole("combobox")[0]); + await user.click(await screen.findByText(label)); +}; + +describe("CreateSearchTools submit payload", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(networking.fetchAvailableSearchProviders).mockResolvedValue({ providers }); + vi.mocked(networking.createSearchTool).mockResolvedValue({ search_tool_id: "st-1" }); + }); + + it("sends every filled field under litellm_params and search_tool_info", async () => { + const user = userEvent.setup(); + renderModal(); + await screen.findByLabelText(/Search Tool Name/); + + await user.type(screen.getByLabelText(/Search Tool Name/), "my-search"); + await pickProvider(user, "Perplexity AI"); + await user.type(screen.getByLabelText(/API Key/), "sk-secret"); + await user.type(screen.getByLabelText(/Description/), "finds things"); + await user.click(screen.getByRole("button", { name: "Add Search Tool" })); + + await waitFor(() => expect(networking.createSearchTool).toHaveBeenCalledTimes(1)); + const [token, payload] = vi.mocked(networking.createSearchTool).mock.calls[0]; + expect(token).toBe("test-token"); + expect(payload).toStrictEqual({ + search_tool_name: "my-search", + litellm_params: { + search_provider: "perplexity", + api_key: "sk-secret", + api_base: undefined, + timeout: undefined, + max_retries: undefined, + }, + search_tool_info: { description: "finds things" }, + }); + expect(JSON.stringify(payload)).toBe( + '{"search_tool_name":"my-search","litellm_params":{"search_provider":"perplexity","api_key":"sk-secret"},"search_tool_info":{"description":"finds things"}}', + ); + }); + + it("omits untouched optional fields from the wire body instead of sending empty strings", async () => { + const user = userEvent.setup(); + renderModal(); + await screen.findByLabelText(/Search Tool Name/); + + await user.type(screen.getByLabelText(/Search Tool Name/), "minimal"); + await pickProvider(user, "Tavily Search"); + await user.click(screen.getByRole("button", { name: "Add Search Tool" })); + + await waitFor(() => expect(networking.createSearchTool).toHaveBeenCalledTimes(1)); + const payload = vi.mocked(networking.createSearchTool).mock.calls[0][1]; + expect(JSON.stringify(payload)).toBe( + '{"search_tool_name":"minimal","litellm_params":{"search_provider":"tavily"}}', + ); + }); + + it("submits on Enter from the search tool name field", async () => { + const user = userEvent.setup(); + renderModal(); + await screen.findByLabelText(/Search Tool Name/); + + await pickProvider(user, "Perplexity AI"); + await user.type(screen.getByLabelText(/Search Tool Name/), "enter-tool{Enter}"); + + await waitFor(() => expect(networking.createSearchTool).toHaveBeenCalledTimes(1)); + expect(vi.mocked(networking.createSearchTool).mock.calls[0][1]).toMatchObject({ search_tool_name: "enter-tool" }); + }); + + it("still creates the tool when Test Connection is clicked, as the untyped Tremor button did", async () => { + const user = userEvent.setup(); + renderModal(); + await screen.findByLabelText(/Search Tool Name/); + + await user.type(screen.getByLabelText(/Search Tool Name/), "probe-tool"); + await pickProvider(user, "Perplexity AI"); + await user.type(screen.getByLabelText(/API Key/), "sk-secret"); + await user.click(screen.getByRole("button", { name: "Test Connection" })); + + await waitFor(() => expect(networking.createSearchTool).toHaveBeenCalledTimes(1)); + expect(vi.mocked(networking.createSearchTool).mock.calls[0][1]).toMatchObject({ search_tool_name: "probe-tool" }); + }); + + it("blocks submit and keeps the antd validation messages when required fields are empty", async () => { + const user = userEvent.setup(); + renderModal(); + await screen.findByLabelText(/Search Tool Name/); + + await user.click(screen.getByRole("button", { name: "Add Search Tool" })); + + expect(await screen.findByText("Please enter a search tool name")).toBeInTheDocument(); + expect(screen.getByText("Please select a search provider")).toBeInTheDocument(); + expect(networking.createSearchTool).not.toHaveBeenCalled(); + }); + + it("rejects a name with characters outside the allowed pattern", async () => { + const user = userEvent.setup(); + renderModal(); + await screen.findByLabelText(/Search Tool Name/); + + await user.type(screen.getByLabelText(/Search Tool Name/), "bad name!"); + await pickProvider(user, "Perplexity AI"); + await user.click(screen.getByRole("button", { name: "Add Search Tool" })); + + expect( + await screen.findByText("Name can only contain letters, numbers, hyphens, and underscores"), + ).toBeInTheDocument(); + expect(networking.createSearchTool).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx index 2af09bebb2e..86baba7470a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/search-tools/_components/CreateSearchTools.tsx @@ -1,13 +1,32 @@ import { isAdminRole } from "@/utils/roles"; -import { InfoCircleOutlined } from "@ant-design/icons"; import { useQuery } from "@tanstack/react-query"; -import { Button, TextInput } from "@tremor/react"; -import { Form, Input, Modal, Select, Tooltip, Typography } from "antd"; -import React, { useState } from "react"; +import { Modal } from "antd"; +import { CircleHelp } from "lucide-react"; +import React, { useCallback, useMemo, useState } from "react"; +import { useWatch } from "react-hook-form"; +import { z } from "zod/v4"; import { Logo } from "@/components/molecules/logo/Logo"; import { toast } from "@/lib/toast"; import { createSearchTool, fetchAvailableSearchProviders } from "@/components/networking"; +import { PasswordInput } from "@/components/shared/PasswordInput"; +import { FieldGroup } from "@/components/shared/form/field"; +import { FormField } from "@/components/shared/form/FormField"; +import { Button } from "@/components/ui/button"; +import { + Combobox, + ComboboxContent, + ComboboxEmpty, + ComboboxInput, + ComboboxItem, + ComboboxList, +} from "@/components/ui/combobox"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import { useZodForm } from "@/lib/forms/useZodForm"; import SearchConnectionTest from "./SearchConnectionTest"; +import { buildSearchToolPayload } from "./searchToolPayload"; import { AvailableSearchProvider, SearchTool } from "./types"; import dataforseoLogo from "../../../../../public/assets/logos/dataforseo.png"; import exaAiLogo from "../../../../../public/assets/logos/exa_ai.png"; @@ -17,8 +36,6 @@ import parallelAiLogo from "../../../../../public/assets/logos/parallel_ai.png"; import perplexityLogo from "../../../../../public/assets/logos/perplexity.png"; import tavilyLogo from "../../../../../public/assets/logos/tavily.png"; -const { TextArea } = Input; - const searchProviderLogoMap: Record = { perplexity: perplexityLogo.src, tavily: tavilyLogo.src, @@ -41,6 +58,32 @@ export const SearchProviderLabel: React.FC = ({ provid
); +const createSearchToolShape = { + search_tool_name: z + .string() + .min(1, "Please enter a search tool name") + .regex(/^[a-zA-Z0-9_-]+$/, "Name can only contain letters, numbers, hyphens, and underscores"), + search_provider: z.string().min(1, "Please select a search provider"), + api_key: z.string().optional(), + description: z.string().optional(), +}; + +const createSearchToolSchema = z.object(createSearchToolShape); + +type CreateSearchToolFormValues = z.infer; + +const EMPTY_VALUES: CreateSearchToolFormValues = { search_tool_name: "", search_provider: "" }; + +const labelWithHint = (label: string, hint: string): React.ReactNode => ( + <> + {label} + + } /> + {hint} + + +); + interface CreateSearchToolProps { userRole: string; accessToken: string | null; @@ -56,14 +99,16 @@ const CreateSearchTool: React.FC = ({ isModalVisible, setModalVisible, }) => { - const [form] = Form.useForm(); + const form = useZodForm(createSearchToolSchema, { defaultValues: EMPTY_VALUES }); const [isLoading, setIsLoading] = useState(false); - const [formValues, setFormValues] = useState>({}); const [isTestModalVisible, setIsTestModalVisible] = useState(false); const [isTestingConnection, setIsTestingConnection] = useState(false); const [connectionTestId, setConnectionTestId] = useState(""); + const [watchedProvider, watchedApiKey] = useWatch({ + control: form.control, + name: ["search_provider", "api_key"], + }); - // Fetch available search providers const { data: providersResponse, isLoading: isLoadingProviders } = useQuery({ queryKey: ["searchProviders"], queryFn: () => { @@ -73,34 +118,28 @@ const CreateSearchTool: React.FC = ({ enabled: !!accessToken && isModalVisible, }) as { data: { providers: AvailableSearchProvider[] }; isLoading: boolean }; - const availableProviders = providersResponse?.providers || []; + const availableProviders = providersResponse?.providers; + const providerNames = useMemo( + () => (availableProviders ?? []).map((provider) => provider.provider_name), + [availableProviders], + ); + const providerLabel = useCallback( + (providerName: string) => + (availableProviders ?? []).find((provider) => provider.provider_name === providerName)?.ui_friendly_name ?? + providerName, + [availableProviders], + ); - const handleCreate = async (formValues: Record) => { + const handleCreate = async (formValues: CreateSearchToolFormValues) => { setIsLoading(true); try { - // Prepare the payload - const payload = { - search_tool_name: formValues.search_tool_name, - litellm_params: { - search_provider: formValues.search_provider, - api_key: formValues.api_key, - api_base: formValues.api_base, - timeout: formValues.timeout ? parseFloat(formValues.timeout) : undefined, - max_retries: formValues.max_retries ? parseInt(formValues.max_retries) : undefined, - }, - search_tool_info: formValues.description - ? { - description: formValues.description, - } - : undefined, - }; + const payload = buildSearchToolPayload(formValues); if (accessToken != null) { const response = await createSearchTool(accessToken, payload); toast.success("Search tool created successfully"); - form.resetFields(); - setFormValues({}); + form.reset(EMPTY_VALUES); setModalVisible(false); onCreateSuccess(response); } @@ -112,32 +151,21 @@ const CreateSearchTool: React.FC = ({ }; const handleCancel = () => { - form.resetFields(); - setFormValues({}); + form.reset(EMPTY_VALUES); setModalVisible(false); }; const handleTestConnection = async () => { - try { - // Validate required fields for testing - await form.validateFields(["search_provider", "api_key"]); - - setIsTestingConnection(true); - // Generate a new test ID (using timestamp for uniqueness) - setConnectionTestId(`test-${Date.now()}`); - // Show the modal with the fresh test - setIsTestModalVisible(true); - } catch (error) { + const isValid = await form.trigger(["search_provider", "api_key"]); + if (!isValid) { toast.error("Please fill in Search Provider and API Key before testing"); + return; } - }; - // Clear formValues when modal closes to reset - React.useEffect(() => { - if (!isModalVisible) { - setFormValues({}); - } - }, [isModalVisible]); + setIsTestingConnection(true); + setConnectionTestId(`test-${Date.now()}`); + setIsTestModalVisible(true); + }; if (!isAdminRole(userRole)) { return null; @@ -146,9 +174,9 @@ const CreateSearchTool: React.FC = ({ return ( +
🔍 -

Add New Search Tool

+

Add New Search Tool

} open={isModalVisible} @@ -162,129 +190,132 @@ const CreateSearchTool: React.FC = ({ }} >
-
setFormValues(allValues)} - layout="vertical" - className="space-y-6" - > -
- - Search Tool Name - - - - - } - name="search_tool_name" - rules={[ - { required: true, message: "Please enter a search tool name" }, - { - pattern: /^[a-zA-Z0-9_-]+$/, - message: "Name can only contain letters, numbers, hyphens, and underscores", - }, - ]} - > - - - - - Search Provider - - - - - } - name="search_provider" - rules={[{ required: true, message: "Please select a search provider" }]} - > - + )} + + + + {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( + onChange(provider ?? "")} > - - - ))} - - + + No matching search providers + + {(providerName: string) => ( + + + + )} + + + + )} + - - API Key - - - - - } - name="api_key" - rules={[{ required: false, message: "Please enter an API key" }]} - > - - + + {({ ref, value, ...field }) => ( + + )} + - Description (Optional)} - name="description" - > -