From 498c4254ead47876814b8a4f9b2fce485b943a37 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sun, 7 Dec 2025 20:53:51 +0800 Subject: [PATCH 001/388] fix: Return 403 exception when calling GET responses api --- litellm/proxy/auth/auth_checks.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index fc79a4d3591..309bd577606 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -402,13 +402,14 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool: - user_route: str - the route the user is trying to call - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user. """ + from starlette.routing import compile_path for allowed_route in allowed_routes: - if ( - allowed_route in LiteLLMRoutes.__members__ - and user_route in LiteLLMRoutes[allowed_route].value - ): - return True + if allowed_route in LiteLLMRoutes.__members__: + for template in LiteLLMRoutes[allowed_route].value: + regex, _, _ = compile_path(template) + if regex.match(user_route): + return True elif allowed_route == user_route: return True return False From 4ab58619ad56d93eb4add67bfd6064c50f449fa1 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sun, 14 Dec 2025 17:55:13 +0800 Subject: [PATCH 002/388] fix: added new step into rotate master key function for processing credentials table --- .../proxy/credential_endpoints/endpoints.py | 9 ++--- .../key_management_endpoints.py | 34 +++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py index 647abb73648..9f228bb1184 100644 --- a/litellm/proxy/credential_endpoints/endpoints.py +++ b/litellm/proxy/credential_endpoints/endpoints.py @@ -21,11 +21,11 @@ router = APIRouter() class CredentialHelperUtils: @staticmethod - def encrypt_credential_values(credential: CredentialItem) -> CredentialItem: + def encrypt_credential_values(credential: CredentialItem, new_encryption_key: Optional[str] = None) -> CredentialItem: """Encrypt values in credential.credential_values and add to DB""" encrypted_credential_values = {} for key, value in (credential.credential_values or {}).items(): - encrypted_credential_values[key] = encrypt_value_helper(value) + encrypted_credential_values[key] = encrypt_value_helper(value, new_encryption_key) # Return a new object to avoid mutating the caller's credential, which # is kept in memory and should remain unencrypted. @@ -246,7 +246,7 @@ async def delete_credential( def update_db_credential( - db_credential: CredentialItem, updated_patch: CredentialItem + db_credential: CredentialItem, updated_patch: CredentialItem, new_encryption_key: Optional[str] = None ) -> CredentialItem: """ Update a credential in the DB. @@ -258,7 +258,8 @@ def update_db_credential( ) encrypted_credential = CredentialHelperUtils.encrypt_credential_values( - updated_patch + updated_patch, + new_encryption_key, ) # update model name if encrypted_credential.credential_name: diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index da44bda791d..8ea3122ce01 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -2539,6 +2539,40 @@ async def _rotate_master_key( new_master_key=new_master_key, ) + # 5. process credentials table + try: + credentials = await prisma_client.db.litellm_credentialstable.find_many() + except Exception: + credentials = None + if credentials: + from litellm.proxy.credential_endpoints.endpoints import update_db_credential + + for cred in credentials: + try: + decrypted_cred = proxy_config.decrypt_credentials(cred) + encrypted_cred = update_db_credential( + db_credential=cred, + updated_patch=decrypted_cred, + new_encryption_key=new_master_key, + ) + credential_object_jsonified = jsonify_object(encrypted_cred.model_dump()) + await prisma_client.db.litellm_credentialstable.update( + where={"credential_name": cred.credential_name}, + data={ + **credential_object_jsonified, + "updated_by": user_api_key_dict.user_id, + }, + ) + except Exception as e: + verbose_proxy_logger.error( + f"Failed to re-encrypt credential {cred.credential_name}: {str(e)}" + ) + # Continue with next credential instead of failing entire rotation + continue + verbose_proxy_logger.debug( + f"Successfully re-encrypted {len(credentials)} credentials with new master key" + ) + def get_new_token(data: Optional[RegenerateKeyRequest]) -> str: if data and data.new_key is not None: From f4f5ea85dfe7eff7130724b1d7331354468f5ce8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 18 Dec 2025 14:42:41 +0530 Subject: [PATCH 003/388] Add redisvl in requirements.txt --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c36f94f0752..2fb6c52cfd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,8 @@ starlette==0.49.1 # starlette fastapi dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep uvicorn==0.31.1 # server dep -gunicorn==23.0.0 # server dep +gunicorn==23.0.0 # server depredisvl +redisvl==0.4.1 # redis semantic cache fastuuid==0.13.5 # for uuid4 uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls From 5705aaebbc3247cb1666dec0288ab7b5507b2c79 Mon Sep 17 00:00:00 2001 From: Rens Date: Thu, 18 Dec 2025 15:11:58 +0200 Subject: [PATCH 004/388] Fix Gemini 3 imgs in tool response --- .../prompt_templates/factory.py | 7 +-- ...llm_core_utils_prompt_templates_factory.py | 51 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 652692c7b8d..9afea83ef7b 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1496,9 +1496,10 @@ def convert_to_gemini_tool_call_result( content_type = content.get("type", "") if content_type == "text": content_str += content.get("text", "") - elif content_type == "input_image": - # Extract image for inline_data (for Computer Use screenshots) - image_url = content.get("image_url", "") + elif content_type in ("input_image", "image_url"): + # Extract image for inline_data (for Computer Use screenshots and tool results) + image_url_data = content.get("image_url", "") + image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data if image_url: # Convert image to base64 blob format for Gemini diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 41ac893b4d7..c8fe6efeaa1 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -497,6 +497,57 @@ def test_convert_gemini_messages(): ) +def test_convert_gemini_tool_call_result_with_image_url(): + """ + Test that image_url content type in tool results is handled correctly for Gemini. + Fixes: https://github.com/BerriAI/litellm/issues/18187 + """ + from litellm.litellm_core_utils.prompt_templates.factory import ( + convert_to_gemini_tool_call_result, + ) + from litellm.types.llms.openai import ChatCompletionToolMessage + + # Test with string image_url format + message_str_format = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_123", + content=[{"type": "image_url", "image_url": "data:image/jpeg;base64,/9j/4AAQ"}], + ) + last_message_with_tool_calls = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "index": 0, + "function": {"name": "get_image", "arguments": "{}"}, + } + ], + } + + result = convert_to_gemini_tool_call_result( + message=message_str_format, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + # Should have inline_data for the image + assert isinstance(result, list) and any("inline_data" in p for p in result) + + # Test with dict image_url format (OpenAI standard) + message_dict_format = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_456", + content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}}], + ) + last_message_with_tool_calls["tool_calls"][0]["id"] = "call_456" + + result2 = convert_to_gemini_tool_call_result( + message=message_dict_format, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + assert isinstance(result2, list) and any("inline_data" in p for p in result2) + + def test_bedrock_tools_unpack_defs(): """ Test that the unpack_defs method handles nested $ref inside anyOf items correctly From 7ddab06bedba0d6151309c308a38cbfa13588dff Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 11:35:20 +0800 Subject: [PATCH 005/388] fix: fixed the issue of handling root paths when processing Discovery protected resource metadata and authorization server metadata URLs. --- .../mcp_server/discoverable_endpoints.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..4b6020f582b 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -15,6 +15,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import ( ) from litellm.proxy.common_utils.http_parsing_utils import _read_request_body from litellm.types.mcp_server.mcp_server_manager import MCPServer +from litellm.proxy.utils import get_server_root_path router = APIRouter( tags=["mcp"], @@ -381,7 +382,18 @@ async def callback(code: str, state: str): # ------------------------------ # Optional .well-known endpoints for MCP + OAuth discovery # ------------------------------ -@router.get("/.well-known/oauth-protected-resource/{mcp_server_name}/mcp") +""" + Per SEP-985, the client MUST: + 1. Try resource_metadata from WWW-Authenticate header (if present) + 2. Fall back to path-based well-known URI: /.well-known/oauth-protected-resource/{path} + ( + If the resource identifier value contains a path or query component, any terminating slash (/) + following the host component MUST be removed before inserting /.well-known/ and the well-known + URI path suffix between the host component and the path(include root path) and/or query components. + https://datatracker.ietf.org/doc/html/rfc9728#section-3.1) + 3. Fall back to root-based well-known URI: /.well-known/oauth-protected-resource +""" +@router.get(f"/.well-known/oauth-protected-resource{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}/mcp") @router.get("/.well-known/oauth-protected-resource") async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None @@ -403,8 +415,15 @@ async def oauth_protected_resource_mcp( ), # this is what Claude will call } - -@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}") +""" + https://datatracker.ietf.org/doc/html/rfc8414#section-3.1 + RFC 8414: Path-aware OAuth discovery + If the issuer identifier value contains a path component, any + terminating "/" MUST be removed before inserting "/.well-known/" and + the well-known URI suffix between the host component and the path(include root path) + component. +""" +@router.get(f"/.well-known/oauth-authorization-server{'' if get_server_root_path() == '/' else get_server_root_path()}/{{mcp_server_name}}") @router.get("/.well-known/oauth-authorization-server") async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None From 3a2ab6b0d12be8863d2a7604a1f3e8ab5721b521 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 11:55:12 +0800 Subject: [PATCH 006/388] fix: added additional grant type into oauth_authorization_server response for fixing mcp auth register bad request issue --- .../proxy/_experimental/mcp_server/discoverable_endpoints.py | 2 +- .../_experimental/mcp_server/test_discoverable_endpoints.py | 3 ++- ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffa17a5b7c4..5433196dfe3 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -428,7 +428,7 @@ async def oauth_authorization_server_mcp( "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], - "grant_types_supported": ["authorization_code"], + "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 6df9abd3fee..30f3d55f028 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -354,7 +354,7 @@ async def test_register_client_remote_registration_success(): request_payload = { "client_name": "Litellm Proxy", - "grant_types": ["authorization_code"], + "grant_types": ["authorization_code", "refresh_token"], "response_types": ["code"], "token_endpoint_auth_method": "client_secret_post", } @@ -603,6 +603,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): assert response["authorization_endpoint"].startswith("https://litellm.example.com/") assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") + assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx index 9600c962564..a62d8baa75b 100644 --- a/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx +++ b/ui/litellm-dashboard/src/hooks/useMcpOAuthFlow.tsx @@ -136,7 +136,7 @@ export const useMcpOAuthFlow = ({ if (!hasPreconfiguredCredentials) { const registration = await registerMcpOAuthClient(accessToken, serverId, { client_name: temporaryPayload.alias || temporaryPayload.server_name || serverId, - grant_types: ["authorization_code"], + grant_types: ["authorization_code", "refresh_token"], response_types: ["code"], token_endpoint_auth_method: temporaryPayload.credentials && temporaryPayload.credentials.client_secret ? "client_secret_post" : "none", From 684fba42eaaf6a4d47795e56fd668b8d46b01525 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 13:22:29 +0800 Subject: [PATCH 007/388] fix: added RFC RECOMMENDED property(scopes_supported) to protected resource and authorization server metadata --- .../mcp_server/discoverable_endpoints.py | 17 +++++- .../mcp_server/test_discoverable_endpoints.py | 54 ++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index d6fe3f2b9cf..ded591a8f53 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -398,8 +398,14 @@ async def callback(code: str, state: str): async def oauth_protected_resource_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) return { "authorization_servers": [ ( @@ -413,6 +419,7 @@ async def oauth_protected_resource_mcp( if mcp_server_name else f"{request_base_url}/mcp" ), # this is what Claude will call + "scopes_supported": mcp_server.scopes if mcp_server else [], } """ @@ -428,6 +435,9 @@ async def oauth_protected_resource_mcp( async def oauth_authorization_server_mcp( request: Request, mcp_server_name: Optional[str] = None ): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) # Get the correct base URL considering X-Forwarded-* headers request_base_url = get_request_base_url(request) @@ -442,16 +452,21 @@ async def oauth_authorization_server_mcp( else f"{request_base_url}/token" ) + mcp_server: Optional[MCPServer] = None + if mcp_server_name: + mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name) + return { "issuer": request_base_url, # point to your proxy "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "response_types_supported": ["code"], + "scopes_supported": mcp_server.scopes if mcp_server else [], "grant_types_supported": ["authorization_code", "refresh_token"], "code_challenge_methods_supported": ["S256"], "token_endpoint_auth_methods_supported": ["client_secret_post"], # Claude expects a registration endpoint, even if we just fake it - "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register", + "registration_endpoint": f"{request_base_url}/{mcp_server_name}/register" if mcp_server_name else f"{request_base_url}/register", } diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 30f3d55f028..4c5723b8284 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -556,9 +556,33 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_protected_resource_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -568,13 +592,14 @@ async def test_oauth_protected_resource_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_protected_resource_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs assert response["authorization_servers"][0].startswith( "https://litellm.example.com/" ) + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio @@ -584,9 +609,33 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( oauth_authorization_server_mcp, ) + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + from litellm.types.mcp import MCPAuth + from litellm.types.mcp_server.mcp_server_manager import MCPServer + from litellm.proxy._types import MCPTransport from fastapi import Request except ImportError: pytest.skip("MCP discoverable endpoints not available") + # Clear registry + global_mcp_server_manager.registry.clear() + + # Create mock OAuth2 server + oauth2_server = MCPServer( + server_id="test_oauth_server", + name="test_oauth", + server_name="test_oauth", + alias="test_oauth", + transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, + client_id="test_client_id", + client_secret="test_client_secret", + authorization_url="https://provider.com/oauth/authorize", + token_url="https://provider.com/oauth/token", + scopes=["read", "write"], + ) + global_mcp_server_manager.registry[oauth2_server.server_id] = oauth2_server # Mock request with http base_url but X-Forwarded-Proto: https mock_request = MagicMock(spec=Request) @@ -596,7 +645,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): # Call the endpoint response = await oauth_authorization_server_mcp( request=mock_request, - mcp_server_name="test_server", + mcp_server_name="test_oauth", ) # Verify response uses HTTPS URLs @@ -604,6 +653,7 @@ async def test_oauth_authorization_server_respects_x_forwarded_proto(): assert response["token_endpoint"].startswith("https://litellm.example.com/") assert response["registration_endpoint"].startswith("https://litellm.example.com/") assert response["grant_types_supported"] == ["authorization_code", "refresh_token"] + assert response["scopes_supported"] == oauth2_server.scopes @pytest.mark.asyncio From 0306f02e74d7fff462fb727639a60e5ae11d64e4 Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 14:13:18 +0800 Subject: [PATCH 008/388] fix: removed initialize the tool name to MCP server name mapping(oauth2) on startup for avoiding 401 error --- .../mcp_server/mcp_server_manager.py | 3 +++ .../mcp_server/test_mcp_server_manager.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 8c9d8630457..c2215efe9d0 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1913,6 +1913,9 @@ class MCPServerManager: Note: This now handles prefixed tool names """ for server in self.get_registry().values(): + if server.auth_type == MCPAuth.oauth2: + # Skip OAuth2 servers for now as they may require user-specific tokens + continue tools = await self._get_tools_from_server(server) for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7a6e5ad17f6..c0ded9c728c 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -536,7 +536,26 @@ class TestMCPServerManager: assert ( server.registration_url == "https://discovered.example.com/register" ) + @pytest.mark.asyncio + async def test_config_oauth_initialize_tool_name_to_mcp_server_name_mapping(self): + manager = MCPServerManager() + config = { + "example": { + "url": "https://example.com/mcp", + "transport": MCPTransport.http, + "auth_type": MCPAuth.oauth2, + "scopes": ["config"], + "authorization_url": "https://config.example.com/auth", + } + } + + await manager.load_servers_from_config(config) + + # Initialize the tool mapping + await manager._initialize_tool_name_to_mcp_server_name_mapping() + assert manager.tool_name_to_mcp_server_name_mapping == {} + @pytest.mark.asyncio async def test_list_tools_handles_missing_server_alias(self): """Test that list_tools handles servers without alias gracefully""" From 36a369a747fccedb87e12cf26996aebfc221f57e Mon Sep 17 00:00:00 2001 From: Eric84626 Date: Sat, 20 Dec 2025 14:28:52 +0800 Subject: [PATCH 009/388] fix: upgraded mcp sdk depency version for fixing ClosedResourceError --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f222acc46e6..cb12a658814 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.21.2 ; python_version >= "3.10" # for MCP server +mcp==1.25.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging From 9002f75277228c0723d0d503f71f1f1a5de4638f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 20 Dec 2025 11:01:41 -0600 Subject: [PATCH 010/388] Require auth for MCP connection test --- .../mcp_server/rest_endpoints.py | 8 +-- .../mcp_server/test_rest_endpoints.py | 55 +++++++++++++++++++ 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 032331ece02..92c390f0a64 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,5 +1,4 @@ import importlib -import traceback from typing import Dict, List, Optional, Union from fastapi import APIRouter, Depends, Query, Request @@ -329,16 +328,15 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True) - stack_trace = traceback.format_exc() return { "status": "error", - "message": f"An internal error has occurred: {str(e)}", - "stack_trace": stack_trace, + "message": "An internal error has occurred while testing the MCP server.", } - @router.post("/test/connection") + @router.post("/test/connection", dependencies=[Depends(user_api_key_auth)]) async def test_connection( request: NewMCPServerRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Test if we can connect to the provided MCP server before adding it diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index a0c09663a88..85ec807b1ff 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -7,6 +7,7 @@ from litellm.proxy._experimental.mcp_server import rest_endpoints from litellm.proxy._experimental.mcp_server.auth import ( user_api_key_auth_mcp as auth_mcp, ) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy._types import NewMCPServerRequest, UserAPIKeyAuth from litellm.types.mcp import MCPAuth @@ -31,6 +32,60 @@ def _build_request(headers: Optional[Dict[str, str]] = None) -> Request: return Request(scope, receive=receive) +def _get_route(path: str, method: str): + for route in rest_endpoints.router.routes: + if getattr(route, "path", None) == path and method in getattr( + route, "methods", set() + ): + return route + raise AssertionError(f"Route {method} {path} not found") + + +def _route_has_dependency(route, dependency) -> bool: + if any( + getattr(dep, "dependency", None) == dependency + for dep in getattr(route, "dependencies", []) + ): + return True + dependant = getattr(route, "dependant", None) + if dependant is None: + return False + return any(getattr(dep, "call", None) == dependency for dep in dependant.dependencies) + + +@pytest.mark.asyncio +async def test_execute_with_mcp_client_redacts_stack_trace(monkeypatch): + def fake_create_client(*args, **kwargs): + return object() + + monkeypatch.setattr( + rest_endpoints.global_mcp_server_manager, + "_create_mcp_client", + fake_create_client, + ) + + async def failing_operation(client): + raise RuntimeError("boom") + + payload = NewMCPServerRequest( + server_name="example", + url="https://example.com", + auth_type=MCPAuth.none, + ) + + result = await rest_endpoints._execute_with_mcp_client( + payload, failing_operation + ) + + assert result["status"] == "error" + assert "stack_trace" not in result + + +def test_test_connection_requires_auth_dependency(): + route = _get_route("/mcp-rest/test/connection", "POST") + assert _route_has_dependency(route, user_api_key_auth) + + @pytest.mark.asyncio async def test_test_tools_list_forwards_mcp_auth_header(monkeypatch): """Ensure credential-based auth forwards the auth_value to the MCP client.""" From acce6b9c83f143116038b49be710398802cd540e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 20 Dec 2025 15:56:46 -0600 Subject: [PATCH 011/388] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../proxy/_experimental/mcp_server/test_rest_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index 85ec807b1ff..31ab4afb631 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -82,7 +82,7 @@ async def test_execute_with_mcp_client_redacts_stack_trace(monkeypatch): def test_test_connection_requires_auth_dependency(): - route = _get_route("/mcp-rest/test/connection", "POST") + route = _get_route("/test/connection", "POST") assert _route_has_dependency(route, user_api_key_auth) From 34b500c7f5b24b6dd24c9c286081aa07c56b6d4b Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Mon, 22 Dec 2025 11:21:59 +0900 Subject: [PATCH 012/388] feat: support MCP stdio header env overrides --- docs/my-website/docs/mcp.md | 27 +++++- .../mcp_server/mcp_server_manager.py | 62 ++++++++++++- .../mcp_server/rest_endpoints.py | 58 ++++++++---- .../proxy/_experimental/mcp_server/server.py | 6 ++ tests/mcp_tests/test_mcp_server.py | 16 +++- .../mcp_server/test_mcp_server.py | 58 ++++++++++-- .../mcp_server/test_mcp_server_manager.py | 93 +++++++++++++++++-- .../mcp_server/test_rest_endpoints.py | 16 +++- 8 files changed, 295 insertions(+), 41 deletions(-) diff --git a/docs/my-website/docs/mcp.md b/docs/my-website/docs/mcp.md index f9c9cbb4562..a70e3d24188 100644 --- a/docs/my-website/docs/mcp.md +++ b/docs/my-website/docs/mcp.md @@ -746,8 +746,33 @@ curl --location 'http://localhost:4000/github_mcp/mcp' \ 3. **Header Forwarding**: LiteLLM automatically forwards matching headers to the backend MCP server 4. **Authentication**: The backend MCP server receives both the configured auth headers and the custom headers ---- +### Passing Request Headers to STDIO env Vars + +If your stdio MCP server needs per-request credentials, you can map HTTP headers from the client request directly into the environment for the launched stdio process. Reference the header name in the env value using the `${X-HEADER_NAME}` syntax. LiteLLM will read that header from the incoming request and set the env var before starting the command. + +```json title="Forward X-GITHUB_PERSONAL_ACCESS_TOKEN header to stdio env" showLineNumbers +{ + "mcpServers": { + "github": { + "command": "docker", + "args": [ + "run", + "-i", + "--rm", + "-e", + "GITHUB_PERSONAL_ACCESS_TOKEN", + "ghcr.io/github/github-mcp-server" + ], + "env": { + "GITHUB_PERSONAL_ACCESS_TOKEN": "${X-GITHUB_PERSONAL_ACCESS_TOKEN}" + } + } + } +} +``` + +In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable. ## Using your MCP with client side credentials diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 8c9d8630457..2260649e8b2 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -84,6 +84,8 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]: class MCPServerManager: + _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") + def __init__(self): self.registry: Dict[str, MCPServer] = {} self.config_mcp_servers: Dict[str, MCPServer] = {} @@ -671,11 +673,39 @@ class MCPServerManager: ######################################################### # Methods that call the upstream MCP servers ######################################################### + def _build_stdio_env( + self, + server: MCPServer, + raw_headers: Optional[Dict[str, str]] = None, + ) -> Optional[Dict[str, str]]: + """Resolve stdio env values, supporting header-driven placeholders.""" + + if server.transport != MCPTransport.stdio or not server.env: + return None + + resolved_env: Dict[str, str] = {} + normalized_headers = {k.lower(): v for k, v in (raw_headers or {}).items()} + + for env_key, env_value in server.env.items(): + stripped_value = env_value.strip() + match = self._STDIO_ENV_TEMPLATE_PATTERN.match(stripped_value) + if match: + header_name = match.group(1) + header_value = normalized_headers.get(header_name.lower()) + if header_value is None: + continue + resolved_env[env_key] = header_value + else: + resolved_env[env_key] = env_value + + return resolved_env + def _create_mcp_client( self, server: MCPServer, mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, + stdio_env: Optional[Dict[str, str]] = None, ) -> MCPClient: """ Create an MCPClient instance for the given server. @@ -692,10 +722,13 @@ class MCPServerManager: # Handle stdio transport if transport == MCPTransport.stdio: # For stdio, we need to get the stdio config from the server + resolved_env = stdio_env if stdio_env is not None else server.env or {} stdio_config: Optional[MCPStdioConfig] = None if server.command and server.args is not None: stdio_config = MCPStdioConfig( - command=server.command, args=server.args, env=server.env or {} + command=server.command, + args=server.args, + env=resolved_env, ) return MCPClient( @@ -725,6 +758,7 @@ class MCPServerManager: mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, add_prefix: bool = True, + raw_headers: Optional[Dict[str, str]] = None, ) -> List[MCPTool]: """ Helper method to get tools from a single MCP server with prefixed names. @@ -751,10 +785,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(server.static_headers) + stdio_env = self._build_stdio_env(server, raw_headers) + client = self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) ## HANDLE OPENAPI TOOLS @@ -784,6 +821,7 @@ class MCPServerManager: mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, add_prefix: bool = True, + raw_headers: Optional[Dict[str, str]] = None, ) -> List[Prompt]: """ Helper method to get prompts from a single MCP server with prefixed names. @@ -807,10 +845,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(server.static_headers) + stdio_env = self._build_stdio_env(server, raw_headers) + client = self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) prompts = await client.list_prompts() @@ -833,6 +874,7 @@ class MCPServerManager: mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, add_prefix: bool = True, + raw_headers: Optional[Dict[str, str]] = None, ) -> List[Resource]: """Fetch available resources from a single MCP server.""" @@ -847,10 +889,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(server.static_headers) + stdio_env = self._build_stdio_env(server, raw_headers) + client = self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) resources = await client.list_resources() @@ -873,6 +918,7 @@ class MCPServerManager: mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, add_prefix: bool = True, + raw_headers: Optional[Dict[str, str]] = None, ) -> List[ResourceTemplate]: """Fetch available resource templates from a single MCP server.""" @@ -887,10 +933,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(server.static_headers) + stdio_env = self._build_stdio_env(server, raw_headers) + client = self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) resource_templates = await client.list_resource_templates() @@ -913,6 +962,7 @@ class MCPServerManager: url: AnyUrl, mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, ) -> ReadResourceResult: """Read resource contents from a specific MCP server.""" @@ -924,10 +974,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(server.static_headers) + stdio_env = self._build_stdio_env(server, raw_headers) + client = self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) return await client.read_resource(url) @@ -939,6 +992,7 @@ class MCPServerManager: arguments: Optional[Dict[str, Any]] = None, mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, extra_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, ) -> GetPromptResult: """Fetch a specific prompt definition from a single MCP server.""" @@ -950,10 +1004,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(server.static_headers) + stdio_env = self._build_stdio_env(server, raw_headers) + client = self._create_mcp_client( server=server, mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) get_prompt_request_params = GetPromptRequestParams( @@ -1742,10 +1799,13 @@ class MCPServerManager: extra_headers = {} extra_headers.update(mcp_server.static_headers) + stdio_env = self._build_stdio_env(mcp_server, raw_headers) + client = self._create_mcp_client( server=mcp_server, mcp_auth_header=server_auth_header, extra_headers=extra_headers, + stdio_env=stdio_env, ) call_tool_params = MCPCallToolRequestParams( diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index 032331ece02..891b52db7af 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -71,12 +71,17 @@ if MCP_AVAILABLE: for tool in tools ] - async def _get_tools_for_single_server(server, server_auth_header): + async def _get_tools_for_single_server( + server, + server_auth_header, + raw_headers: Optional[Dict[str, str]] = None, + ): """Helper function to get tools for a single server.""" tools = await global_mcp_server_manager._get_tools_from_server( server=server, mcp_auth_header=server_auth_header, add_prefix=False, + raw_headers=raw_headers, ) # Filter tools based on allowed_tools configuration @@ -122,6 +127,7 @@ if MCP_AVAILABLE: try: # Extract auth headers from request headers = request.headers + raw_headers_from_request = dict(headers) mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers( headers ) @@ -148,7 +154,7 @@ if MCP_AVAILABLE: try: list_tools_result = await _get_tools_for_single_server( - server, server_auth_header + server, server_auth_header, raw_headers_from_request ) except Exception as e: verbose_logger.exception( @@ -169,7 +175,7 @@ if MCP_AVAILABLE: try: tools_result = await _get_tools_for_single_server( - server, server_auth_header + server, server_auth_header, raw_headers_from_request ) list_tools_result.extend(tools_result) except Exception as e: @@ -232,13 +238,13 @@ if MCP_AVAILABLE: # but they weren't being extracted and passed to call_mcp_tool. # This fix ensures auth headers are properly extracted from the HTTP request # and passed through to the MCP server for authentication. + headers = request.headers + raw_headers_from_request = dict(headers) mcp_auth_header = MCPRequestHandler._get_mcp_auth_header_from_headers( - request.headers + headers ) mcp_server_auth_headers = ( - MCPRequestHandler._get_mcp_server_auth_headers_from_headers( - request.headers - ) + MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers) ) # Add extracted headers to data dict to pass to call_mcp_tool @@ -246,6 +252,7 @@ if MCP_AVAILABLE: data["mcp_auth_header"] = mcp_auth_header if mcp_server_auth_headers: data["mcp_server_auth_headers"] = mcp_server_auth_headers + data["raw_headers"] = raw_headers_from_request result = await call_mcp_tool(**data) return result @@ -300,6 +307,7 @@ if MCP_AVAILABLE: operation, mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None, oauth2_headers: Optional[Dict[str, str]] = None, + raw_headers: Optional[Dict[str, str]] = None, ): """ Common helper to create MCP client, execute operation, and ensure proper cleanup. @@ -312,17 +320,27 @@ if MCP_AVAILABLE: Operation result or error response """ try: + server_model = MCPServer( + server_id=request.server_id or "", + name=request.alias or request.server_name or "", + url=request.url, + transport=request.transport, + auth_type=request.auth_type, + mcp_info=request.mcp_info, + command=request.command, + args=request.args, + env=request.env, + ) + + stdio_env = global_mcp_server_manager._build_stdio_env( + server_model, raw_headers + ) + client = global_mcp_server_manager._create_mcp_client( - server=MCPServer( - server_id=request.server_id or "", - name=request.alias or request.server_name or "", - url=request.url, - transport=request.transport, - auth_type=request.auth_type, - mcp_info=request.mcp_info, - ), + server=server_model, mcp_auth_header=mcp_auth_header, extra_headers=oauth2_headers, + stdio_env=stdio_env, ) return await operation(client) @@ -338,7 +356,8 @@ if MCP_AVAILABLE: @router.post("/test/connection") async def test_connection( - request: NewMCPServerRequest, + request: Request, + new_mcp_server_request: NewMCPServerRequest, ): """ Test if we can connect to the provided MCP server before adding it @@ -351,7 +370,11 @@ if MCP_AVAILABLE: await client.run_with_session(_noop) return {"status": "ok"} - return await _execute_with_mcp_client(request, _test_connection_operation) + return await _execute_with_mcp_client( + new_mcp_server_request, + _test_connection_operation, + raw_headers=dict(request.headers), + ) @router.post("/test/tools/list") async def test_tools_list( @@ -405,4 +428,5 @@ if MCP_AVAILABLE: _list_tools_operation, mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, + raw_headers=dict(request.headers), ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index bdff60c932b..e00fdbfb930 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -775,6 +775,7 @@ if MCP_AVAILABLE: mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=add_prefix, + raw_headers=raw_headers, ) filtered_tools = filter_tools_by_allowed_tools(tools, server) @@ -854,6 +855,7 @@ if MCP_AVAILABLE: mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=add_prefix, + raw_headers=raw_headers, ) all_prompts.extend(prompts) @@ -912,6 +914,7 @@ if MCP_AVAILABLE: mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=add_prefix, + raw_headers=raw_headers, ) all_resources.extend(resources) @@ -969,6 +972,7 @@ if MCP_AVAILABLE: mcp_auth_header=server_auth_header, extra_headers=extra_headers, add_prefix=add_prefix, + raw_headers=raw_headers, ) ) all_resource_templates.extend(resource_templates) @@ -1392,6 +1396,7 @@ if MCP_AVAILABLE: arguments=arguments, mcp_auth_header=server_auth_header, extra_headers=extra_headers, + raw_headers=raw_headers, ) async def mcp_read_resource( @@ -1440,6 +1445,7 @@ if MCP_AVAILABLE: url=url, mcp_auth_header=server_auth_header, extra_headers=extra_headers, + raw_headers=raw_headers, ) def _get_standard_logging_mcp_tool_call( diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index d3112714a9c..9242dfc75f4 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -812,10 +812,19 @@ async def test_get_tools_from_mcp_servers(): return_value=["server1_id", "server2_id"] ) mock_manager_2.get_mcp_server_by_id = lambda server_id: mock_server_1 if server_id == "server1_id" else mock_server_2 + async def mock_get_tools_side_effect( + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=False, + raw_headers=None, + ): + if server.server_id == "server1_id": + return [mock_tool_1] + return [mock_tool_2] + mock_manager_2._get_tools_from_server = AsyncMock( - side_effect=lambda server, mcp_auth_header=None, extra_headers=None, add_prefix=False: ( - [mock_tool_1] if server.server_id == "server1_id" else [mock_tool_2] - ) + side_effect=mock_get_tools_side_effect ) with patch( @@ -1693,6 +1702,7 @@ async def test_get_tools_for_single_server(): server=mock_server, mcp_auth_header="Bearer test_token", add_prefix=False, + raw_headers=None, ) # Verify the result diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py index 4fc94000d61..a1fbddec586 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -294,6 +294,7 @@ async def test_mcp_get_prompt_success(): arguments={"foo": "bar"}, mcp_auth_header={"Authorization": "token"}, extra_headers={"X-Test": "1"}, + raw_headers=None, ) assert result is prompt_result @@ -349,6 +350,7 @@ async def test_mcp_read_resource_success(): url="https://example.com/resource", mcp_auth_header={"Authorization": "token"}, extra_headers={"X-Test": "1"}, + raw_headers=None, ) assert result is read_result @@ -428,7 +430,11 @@ async def test_get_tools_from_mcp_servers_continues_when_one_server_fails(): ) async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=True + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=True, + raw_headers=None, ): if server.name == "working_server": # Working server returns tools @@ -524,7 +530,11 @@ async def test_get_tools_from_mcp_servers_handles_all_servers_failing(): ) async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=True + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=True, + raw_headers=None, ): # All servers fail raise Exception(f"Server {server.name} connection failed") @@ -839,13 +849,19 @@ async def test_oauth2_headers_passed_to_mcp_client(): # This will capture the arguments passed to _create_mcp_client captured_client_args = {} - def mock_create_mcp_client(server, mcp_auth_header=None, extra_headers=None): + def mock_create_mcp_client( + server, + mcp_auth_header=None, + extra_headers=None, + stdio_env=None, + ): # Capture the arguments for verification captured_client_args.update( { "server": server, "mcp_auth_header": mcp_auth_header, "extra_headers": extra_headers, + "stdio_env": stdio_env, } ) # Return a mock client that doesn't actually connect @@ -934,7 +950,11 @@ async def test_list_tools_single_server_unprefixed_names(): mock_manager.get_mcp_server_by_id = MagicMock(return_value=server) async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=False + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=False, + raw_headers=None, ): tool = MagicMock() tool.name = f"{server.alias}-toolA" if add_prefix else "toolA" @@ -1006,7 +1026,11 @@ async def test_list_tools_multiple_servers_prefixed_names(): ) async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=True + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=True, + raw_headers=None, ): tool = MagicMock() # When multiple servers, add_prefix should be True -> prefixed names @@ -1147,7 +1171,11 @@ async def test_list_tools_filters_by_key_team_permissions(): mock_manager.get_mcp_server_by_id = lambda server_id: server async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=False + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=False, + raw_headers=None, ): # Return 4 tools, but only 2 should be allowed tool1 = MagicMock() @@ -1248,7 +1276,11 @@ async def test_list_tools_with_team_tool_permissions_inheritance(): mock_manager.get_mcp_server_by_id = lambda server_id: server async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=False + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=False, + raw_headers=None, ): # Return 4 tools tool1 = MagicMock() @@ -1334,7 +1366,11 @@ async def test_list_tools_with_no_tool_permissions_shows_all(): mock_manager.get_mcp_server_by_id = lambda server_id: server async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=False + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=False, + raw_headers=None, ): # Return 3 tools tool1 = MagicMock() @@ -1423,7 +1459,11 @@ async def test_list_tools_strips_prefix_when_matching_permissions(): mock_manager.get_mcp_server_by_id = MagicMock(return_value=server) async def mock_get_tools_from_server( - server, mcp_auth_header=None, extra_headers=None, add_prefix=True + server, + mcp_auth_header=None, + extra_headers=None, + add_prefix=True, + raw_headers=None, ): # Return tools WITH prefix (as they come from MCP server) tool1 = MagicMock() diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 7a6e5ad17f6..ff016a1a130 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -8,6 +8,7 @@ from fastapi import HTTPException # Add the parent directory to the path so we can import litellm sys.path.insert(0, "../../../../../") + import httpx from mcp import ReadResourceResult, Resource from mcp.types import ( @@ -99,6 +100,53 @@ class TestMCPServerManager: assert client.stdio_config["args"] == ["server.js"] assert client.stdio_config["env"] == {"NODE_ENV": "test"} + def test_build_stdio_env_only_accepts_x_prefixed_placeholders(self): + """Ensure only ${X-*} placeholders are substituted from headers.""" + manager = MCPServerManager() + server = MCPServer( + server_id="stdio-server-env", + name="stdio_env", + transport=MCPTransport.stdio, + command="node", + args=["server.js"], + env={ + "PASSTHROUGH": "${X-Test-Header}", + "STATIC": "value", + "IGNORED": "${Not-Allowed}", + }, + ) + + env = manager._build_stdio_env( + server, + raw_headers={ + "x-test-header": "resolved-value", + "x-not-used": "other", + }, + ) + + assert env == { + "PASSTHROUGH": "resolved-value", + "STATIC": "value", + "IGNORED": "${Not-Allowed}", + } + + def test_build_stdio_env_missing_header_skips_entry(self): + """Ensure missing headers drop the placeholder from the resolved env.""" + manager = MCPServerManager() + server = MCPServer( + server_id="stdio-server-env-miss", + name="stdio_env_miss", + transport=MCPTransport.stdio, + command="node", + args=["server.js"], + env={"EXPECTED": "${X-Missing}"}, + ) + + env = manager._build_stdio_env(server, raw_headers={}) + + # When the header isn't provided, the key is omitted entirely + assert env == {} + @pytest.mark.asyncio async def test_list_tools_with_server_specific_auth_headers(self): """Test list_tools method with server-specific auth headers""" @@ -123,7 +171,10 @@ class TestMCPServerManager: # Mock _get_tools_from_server to return different results async def mock_get_tools_from_server( - server, mcp_auth_header=None, mcp_protocol_version=None + server, + mcp_auth_header=None, + mcp_protocol_version=None, + raw_headers=None, ): if server.name == "github": tool1 = MagicMock() @@ -174,7 +225,10 @@ class TestMCPServerManager: # Mock _get_tools_from_server async def mock_get_tools_from_server( - server, mcp_auth_header=None, mcp_protocol_version=None + server, + mcp_auth_header=None, + mcp_protocol_version=None, + raw_headers=None, ): assert mcp_auth_header == "legacy-token" # Should use legacy header tool = MagicMock() @@ -209,7 +263,10 @@ class TestMCPServerManager: # Mock _get_tools_from_server async def mock_get_tools_from_server( - server, mcp_auth_header=None, mcp_protocol_version=None + server, + mcp_auth_header=None, + mcp_protocol_version=None, + raw_headers=None, ): assert ( mcp_auth_header == "server-specific-token" @@ -373,6 +430,7 @@ class TestMCPServerManager: server=server, mcp_auth_header="auth", extra_headers=None, + stdio_env=None, ) mock_client.list_resource_templates.assert_awaited_once() mock_prefix.assert_called_once_with(mock_templates, server, add_prefix=False) @@ -554,7 +612,10 @@ class TestMCPServerManager: # Mock _get_tools_from_server async def mock_get_tools_from_server( - server, mcp_auth_header=None, mcp_protocol_version=None + server, + mcp_auth_header=None, + mcp_protocol_version=None, + raw_headers=None, ): assert ( mcp_auth_header == "server-specific-token" @@ -587,7 +648,11 @@ class TestMCPServerManager: manager.get_mcp_server_by_id = MagicMock(return_value=server) # Mock successful _get_tools_from_server - async def mock_get_tools_from_server(server, mcp_auth_header=None): + async def mock_get_tools_from_server( + server, + mcp_auth_header=None, + raw_headers=None, + ): tool1 = MagicMock() tool1.name = "tool1" tool2 = MagicMock() @@ -621,7 +686,11 @@ class TestMCPServerManager: manager.get_mcp_server_by_id = MagicMock(return_value=server) # Mock failed _get_tools_from_server - async def mock_get_tools_from_server(server, mcp_auth_header=None): + async def mock_get_tools_from_server( + server, + mcp_auth_header=None, + raw_headers=None, + ): raise Exception("Connection timeout") manager._get_tools_from_server = mock_get_tools_from_server @@ -683,7 +752,11 @@ class TestMCPServerManager: manager.get_mcp_server_by_id = mock_get_server_by_id # Mock _get_tools_from_server with different results - async def mock_get_tools_from_server(server, mcp_auth_header=None): + async def mock_get_tools_from_server( + server, + mcp_auth_header=None, + raw_headers=None, + ): if server.server_id == "server1": tool = MagicMock() tool.name = "tool1" @@ -724,7 +797,11 @@ class TestMCPServerManager: manager.get_mcp_server_by_id = MagicMock(return_value=server) # Mock _get_tools_from_server to verify auth header is passed - async def mock_get_tools_from_server(server, mcp_auth_header=None): + async def mock_get_tools_from_server( + server, + mcp_auth_header=None, + raw_headers=None, + ): assert mcp_auth_header == "test-token" tool = MagicMock() tool.name = "tool1" diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py index a0c09663a88..ce38ee59e4d 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_rest_endpoints.py @@ -37,7 +37,13 @@ async def test_test_tools_list_forwards_mcp_auth_header(monkeypatch): captured: dict = {} - async def fake_execute(request, operation, mcp_auth_header=None, oauth2_headers=None): + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): captured["mcp_auth_header"] = mcp_auth_header captured["oauth2_headers"] = oauth2_headers return { @@ -87,7 +93,13 @@ async def test_test_tools_list_extracts_oauth2_headers(monkeypatch): captured: dict = {} - async def fake_execute(request, operation, mcp_auth_header=None, oauth2_headers=None): + async def fake_execute( + request, + operation, + mcp_auth_header=None, + oauth2_headers=None, + raw_headers=None, + ): captured["mcp_auth_header"] = mcp_auth_header captured["oauth2_headers"] = oauth2_headers return { From ca635a5e3826dae8383be4a5004143cab55f115f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 11:15:38 +0530 Subject: [PATCH 013/388] Add supports_response_schema to all supported together ai models --- ...model_prices_and_context_window_backup.json | 18 ++++++++++++++++++ model_prices_and_context_window.json | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f4b42d1fd6e..ab7e5ae8e8c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -25111,6 +25111,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": { @@ -25118,6 +25119,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { @@ -25129,6 +25131,7 @@ "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -25140,6 +25143,7 @@ "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -25162,6 +25166,7 @@ "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-R1": { @@ -25174,6 +25179,7 @@ "output_cost_per_token": 7e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { @@ -25185,6 +25191,7 @@ "source": "https://www.together.ai/models/deepseek-r1-0528-throughput", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V3": { @@ -25197,6 +25204,7 @@ "output_cost_per_token": 1.25e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V3.1": { @@ -25216,6 +25224,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { @@ -25245,6 +25254,7 @@ "output_cost_per_token": 8.5e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { @@ -25254,6 +25264,7 @@ "output_cost_per_token": 5.9e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { @@ -25263,6 +25274,7 @@ "output_cost_per_token": 3.5e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { @@ -25318,6 +25330,7 @@ "source": "https://www.together.ai/models/kimi-k2-instruct", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/openai/gpt-oss-120b": { @@ -25329,6 +25342,7 @@ "source": "https://www.together.ai/models/gpt-oss-120b", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/openai/gpt-oss-20b": { @@ -25340,6 +25354,7 @@ "source": "https://www.together.ai/models/gpt-oss-20b", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/togethercomputer/CodeLlama-34b-Instruct": { @@ -25358,6 +25373,7 @@ "source": "https://www.together.ai/models/glm-4-5-air", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/zai-org/GLM-4.6": { @@ -25394,6 +25410,7 @@ "source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { @@ -25405,6 +25422,7 @@ "source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "tts-1": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f4b42d1fd6e..ab7e5ae8e8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -25111,6 +25111,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen2.5-7B-Instruct-Turbo": { @@ -25118,6 +25119,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": { @@ -25129,6 +25131,7 @@ "source": "https://www.together.ai/models/qwen3-235b-a22b-instruct-2507-fp8", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": { @@ -25140,6 +25143,7 @@ "source": "https://www.together.ai/models/qwen3-235b-a22b-thinking-2507", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": { @@ -25162,6 +25166,7 @@ "source": "https://www.together.ai/models/qwen3-coder-480b-a35b-instruct", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-R1": { @@ -25174,6 +25179,7 @@ "output_cost_per_token": 7e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": { @@ -25185,6 +25191,7 @@ "source": "https://www.together.ai/models/deepseek-r1-0528-throughput", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V3": { @@ -25197,6 +25204,7 @@ "output_cost_per_token": 1.25e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/deepseek-ai/DeepSeek-V3.1": { @@ -25216,6 +25224,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": { @@ -25245,6 +25254,7 @@ "output_cost_per_token": 8.5e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": { @@ -25254,6 +25264,7 @@ "output_cost_per_token": 5.9e-07, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": { @@ -25263,6 +25274,7 @@ "output_cost_per_token": 3.5e-06, "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": { @@ -25318,6 +25330,7 @@ "source": "https://www.together.ai/models/kimi-k2-instruct", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/openai/gpt-oss-120b": { @@ -25329,6 +25342,7 @@ "source": "https://www.together.ai/models/gpt-oss-120b", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/openai/gpt-oss-20b": { @@ -25340,6 +25354,7 @@ "source": "https://www.together.ai/models/gpt-oss-20b", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/togethercomputer/CodeLlama-34b-Instruct": { @@ -25358,6 +25373,7 @@ "source": "https://www.together.ai/models/glm-4-5-air", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/zai-org/GLM-4.6": { @@ -25394,6 +25410,7 @@ "source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { @@ -25405,6 +25422,7 @@ "source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking", "supports_function_calling": true, "supports_parallel_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "tts-1": { From 57e613282ade7c37bc57545ecf6fb8f4bf221d13 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 14:55:15 +0530 Subject: [PATCH 014/388] Re add thought signature as normal feat --- .../gemini/vertex_and_google_ai_studio_gemini.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index b1810b40cf9..a5cc3dca8c1 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1318,13 +1318,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): _tool_response_chunk["provider_specific_fields"] = { # type: ignore "thought_signature": thought_signature } - # Only embed in ID if preview features are enabled - if litellm.enable_preview_features: - _tool_response_chunk[ - "id" - ] = _encode_tool_call_id_with_signature( - _tool_response_chunk["id"] or "", thought_signature - ) + _tool_response_chunk[ + "id" + ] = _encode_tool_call_id_with_signature( + _tool_response_chunk["id"] or "", thought_signature + ) _tools.append(_tool_response_chunk) cumulative_tool_call_idx += 1 if len(_tools) == 0: From de99e30d0d619a4b03939219521a5f051ae49751 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 14:56:00 +0530 Subject: [PATCH 015/388] Add: a new pre call hook for checking tool thought signature in id --- litellm/utils.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index 805fbafcfce..7a1b0a34361 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -568,6 +568,111 @@ def get_dynamic_callbacks( return returned_callbacks +def _is_gemini_model(model: Optional[str], custom_llm_provider: Optional[str]) -> bool: + """ + Check if the target model is a Gemini or Vertex AI Gemini model. + """ + if custom_llm_provider in ["gemini", "vertex_ai", "vertex_ai_beta"]: + # For vertex_ai, check if it's actually a Gemini model + if custom_llm_provider in ["vertex_ai", "vertex_ai_beta"]: + return model is not None and "gemini" in model.lower() + return True + + # Check if model name contains gemini + return model is not None and "gemini" in model.lower() + + +def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str: + """ + Remove thought signature from a tool call ID. + """ + if separator in tool_call_id: + return tool_call_id.split(separator, 1)[0] + return tool_call_id + + +def _process_assistant_message_tool_calls( + msg_copy: dict, thought_signature_separator: str +) -> dict: + """ + Process assistant message to remove thought signatures from tool call IDs. + """ + role = msg_copy.get("role") + tool_calls = msg_copy.get("tool_calls") + + if role == "assistant" and isinstance(tool_calls, list): + new_tool_calls = [] + for tc in tool_calls: + # Handle both dict and Pydantic model tool calls + if hasattr(tc, "model_dump"): + # It's a Pydantic model, convert to dict + tc_dict = tc.model_dump() + elif isinstance(tc, dict): + tc_dict = tc.copy() + else: + new_tool_calls.append(tc) + continue + + # Remove thought signature from ID if present + if isinstance(tc_dict.get("id"), str): + if thought_signature_separator in tc_dict["id"]: + tc_dict["id"] = _remove_thought_signature_from_id( + tc_dict["id"], thought_signature_separator + ) + + new_tool_calls.append(tc_dict) + msg_copy["tool_calls"] = new_tool_calls + + return msg_copy + + +def _process_tool_message_id(msg_copy: dict, thought_signature_separator: str) -> dict: + """ + Process tool message to remove thought signature from tool_call_id. + """ + if msg_copy.get("role") == "tool" and isinstance( + msg_copy.get("tool_call_id"), str + ): + if thought_signature_separator in msg_copy["tool_call_id"]: + msg_copy["tool_call_id"] = _remove_thought_signature_from_id( + msg_copy["tool_call_id"], thought_signature_separator + ) + + return msg_copy + + +def _remove_thought_signatures_from_messages( + messages: List, thought_signature_separator: str +) -> List: + """ + Remove thought signatures from tool call IDs in all messages. + """ + processed_messages = [] + + for msg in messages: + # Handle Pydantic models (convert to dict) + if hasattr(msg, "model_dump"): + msg_dict = msg.model_dump() + elif isinstance(msg, dict): + msg_dict = msg.copy() + else: + # Unknown type, keep as is + processed_messages.append(msg) + continue + + # Process assistant messages with tool_calls + msg_dict = _process_assistant_message_tool_calls( + msg_dict, thought_signature_separator + ) + + # Process tool messages with tool_call_id + msg_dict = _process_tool_message_id(msg_dict, thought_signature_separator) + + processed_messages.append(msg_dict) + + return processed_messages + + def function_setup( # noqa: PLR0915 original_function: str, rules_obj, start_time, *args, **kwargs ): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc. @@ -779,6 +884,58 @@ def function_setup( # noqa: PLR0915 input=buffer.getvalue(), model=model, ) + + ### REMOVE THOUGHT SIGNATURES FROM TOOL CALL IDS FOR NON-GEMINI MODELS ### + # Gemini models embed thought signatures in tool call IDs. When sending + # messages with tool calls to non-Gemini providers, we need to remove these + # signatures to ensure compatibility. + if isinstance(messages, list) and len(messages) > 0: + try: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + ) + from litellm.litellm_core_utils.prompt_templates.factory import ( + THOUGHT_SIGNATURE_SEPARATOR, + ) + + # Get custom_llm_provider to determine target provider + custom_llm_provider = kwargs.get("custom_llm_provider") + + # If custom_llm_provider not in kwargs, try to determine it from the model + if not custom_llm_provider and model: + try: + _, custom_llm_provider, _, _ = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + ) + except Exception: + # If we can't determine the provider, skip this processing + pass + + # Only process if target is NOT a Gemini model + if not _is_gemini_model(model, custom_llm_provider): + verbose_logger.debug( + f"Removing thought signatures from tool call IDs for non-Gemini model: {model}" + ) + + # Process messages to remove thought signatures + processed_messages = _remove_thought_signatures_from_messages( + messages, THOUGHT_SIGNATURE_SEPARATOR + ) + + # Update messages in kwargs or args + if "messages" in kwargs: + kwargs["messages"] = processed_messages + elif len(args) > 1: + args = list(args) + args[1] = processed_messages + args = tuple(args) + + except Exception as e: + # Log the error but don't fail the request + verbose_logger.warning( + f"Error removing thought signatures from tool call IDs: {str(e)}" + ) elif ( call_type == CallTypes.embedding.value or call_type == CallTypes.aembedding.value From 0addab4f63088592fc1b4f40088e33b3270533d6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 14:56:42 +0530 Subject: [PATCH 016/388] Add test for removal of thought signature --- tests/local_testing/test_function_setup.py | 176 ++++++++++++++++++++- 1 file changed, 175 insertions(+), 1 deletion(-) diff --git a/tests/local_testing/test_function_setup.py b/tests/local_testing/test_function_setup.py index 5cc3ce12304..23a82fd7a6e 100644 --- a/tests/local_testing/test_function_setup.py +++ b/tests/local_testing/test_function_setup.py @@ -9,9 +9,10 @@ import os, io sys.path.insert( 0, os.path.abspath("../..") -) # Adds the parent directory to the, system path +) # Adds the parent directory to the system path import pytest, uuid from litellm.utils import function_setup, Rules +from litellm.litellm_core_utils.prompt_templates.factory import THOUGHT_SIGNATURE_SEPARATOR from datetime import datetime @@ -31,3 +32,176 @@ def test_empty_content(): messages=[], litellm_call_id=str(uuid.uuid4()), ) + + +def test_thought_signature_removal_for_non_gemini(): + """ + Test that thought signatures are removed from tool call IDs when sending to non-Gemini models + """ + rules_obj = Rules() + + # Create messages with thought signatures (as would come from Gemini) + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "SF"}' + } + } + ] + }, + { + "role": "tool", + "tool_call_id": f"call_123{THOUGHT_SIGNATURE_SEPARATOR}sig1", + "content": "Sunny, 72°F" + } + ] + + # Call function_setup with OpenAI model (non-Gemini) + logging_obj, kwargs = function_setup( + original_function="acompletion", + rules_obj=rules_obj, + start_time=datetime.now(), + model="gpt-4", + messages=messages, + litellm_call_id=str(uuid.uuid4()), + custom_llm_provider="openai" + ) + + # Verify thought signatures were removed + processed_messages = kwargs["messages"] + assert processed_messages[1]["tool_calls"][0]["id"] == "call_123" + assert processed_messages[2]["tool_call_id"] == "call_123" + assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[1]["tool_calls"][0]["id"] + assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[2]["tool_call_id"] + + +def test_thought_signature_preserved_for_gemini(): + """ + Test that thought signatures are preserved when sending to Gemini models + """ + rules_obj = Rules() + + # Create messages with thought signatures + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"location": "NYC"}' + } + } + ] + }, + { + "role": "tool", + "tool_call_id": f"call_456{THOUGHT_SIGNATURE_SEPARATOR}sig2", + "content": "Rainy, 65°F" + } + ] + + # Call function_setup with Gemini model + logging_obj, kwargs = function_setup( + original_function="acompletion", + rules_obj=rules_obj, + start_time=datetime.now(), + model="gemini-1.5-pro", + messages=messages, + litellm_call_id=str(uuid.uuid4()), + custom_llm_provider="vertex_ai" + ) + + # Verify thought signatures were preserved (messages should be unchanged) + processed_messages = kwargs["messages"] + assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[1]["tool_calls"][0]["id"] + assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[2]["tool_call_id"] + + +def test_thought_signature_removal_with_multiple_tool_calls(): + """ + Test that thought signatures are removed from multiple tool calls + """ + rules_obj = Rules() + + messages = [ + {"role": "user", "content": "Get weather and time"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1", + "type": "function", + "function": {"name": "get_weather", "arguments": "{}"} + }, + { + "id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2", + "type": "function", + "function": {"name": "get_time", "arguments": "{}"} + } + ] + }, + { + "role": "tool", + "tool_call_id": f"call_1{THOUGHT_SIGNATURE_SEPARATOR}sig1", + "content": "Sunny" + }, + { + "role": "tool", + "tool_call_id": f"call_2{THOUGHT_SIGNATURE_SEPARATOR}sig2", + "content": "3:00 PM" + } + ] + + logging_obj, kwargs = function_setup( + original_function="acompletion", + rules_obj=rules_obj, + start_time=datetime.now(), + model="claude-3-opus", + messages=messages, + litellm_call_id=str(uuid.uuid4()), + custom_llm_provider="anthropic" + ) + + processed_messages = kwargs["messages"] + + # Check all tool call IDs are cleaned + assert processed_messages[1]["tool_calls"][0]["id"] == "call_1" + assert processed_messages[1]["tool_calls"][1]["id"] == "call_2" + assert processed_messages[2]["tool_call_id"] == "call_1" + assert processed_messages[3]["tool_call_id"] == "call_2" + + +def test_messages_without_tool_calls_unchanged(): + """ + Test that messages without tool calls pass through unchanged + """ + rules_obj = Rules() + + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there!"} + ] + + logging_obj, kwargs = function_setup( + original_function="acompletion", + rules_obj=rules_obj, + start_time=datetime.now(), + model="gpt-4", + messages=messages, + litellm_call_id=str(uuid.uuid4()), + custom_llm_provider="openai" + ) + + # Messages should be unchanged + assert kwargs["messages"] == messages From 69897bea48e3e5aafac49de6bbd92a4fdef9637b Mon Sep 17 00:00:00 2001 From: Matt Cowger Date: Tue, 23 Dec 2025 02:05:54 -0800 Subject: [PATCH 017/388] Add 5 AI providers using `openai_like` (#18362) * Add 5 AI providers using `openai_like`: * Synthetic.new * Apertis / Stima.tech * NanoGPT * Poe * Chutes.ai * Update additional missing locations --- litellm/constants.py | 16 +++- .../get_llm_provider_logic.py | 15 ++++ litellm/llms/openai_like/providers.json | 37 ++++++++- litellm/types/utils.py | 6 ++ provider_endpoints_support.json | 82 ++++++++++++++++++- 5 files changed, 153 insertions(+), 3 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 511cbafc748..8fd2e307555 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -556,6 +556,11 @@ openai_compatible_endpoints: List = [ "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", + "https://api.synthetic.new/openai/v1", + "https://api.stima.tech/v1", + "https://nano-gpt.com/api/v1", + "https://api.poe.com/v1", + "https://llm.chutes.ai/v1/", "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", @@ -599,12 +604,16 @@ openai_compatible_providers: List = [ "novita", "meta_llama", "publicai", # PublicAI - JSON-configured provider + "synthetic", # Synthetic - JSON-configured provider + "apertis", # Apertis - JSON-configured provider + "nano-gpt", # Nano-GPT - JSON-configured provider + "poe", # Poe - JSON-configured provider + "chutes", # Chutes - JSON-configured provider "featherless_ai", "nscale", "nebius", "dashscope", "moonshot", - "publicai", "v0", "helicone", "morph", @@ -630,6 +639,11 @@ openai_text_completion_compatible_providers: List = ( "dashscope", "moonshot", "publicai", + "synthetic", + "apertis", + "nano-gpt", + "poe", + "chutes", "v0", "lambda_ai", "hyperbolic", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a23fce891b9..03ea7c93978 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -270,6 +270,21 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "platform.publicai.co/v1": custom_llm_provider = "publicai" dynamic_api_key = get_secret_str("PUBLICAI_API_KEY") + elif endpoint == "https://api.synthetic.new/openai/v1": + custom_llm_provider = "synthetic" + dynamic_api_key = get_secret_str("SYNTHETIC_API_KEY") + elif endpoint == "https://api.stima.tech/v1": + custom_llm_provider = "apertis" + dynamic_api_key = get_secret_str("STIMA_API_KEY") + elif endpoint == "https://nano-gpt.com/api/v1": + custom_llm_provider = "nano-gpt" + dynamic_api_key = get_secret_str("NANOGPT_API_KEY") + elif endpoint == "https://api.poe.com/v1": + custom_llm_provider = "poe" + dynamic_api_key = get_secret_str("POE_API_KEY") + elif endpoint == "https://llm.chutes.ai/v1/": + custom_llm_provider = "chutes" + dynamic_api_key = get_secret_str("CHUTES_API_KEY") elif endpoint == "https://api.v0.dev/v1": custom_llm_provider = "v0" dynamic_api_key = get_secret_str("V0_API_KEY") diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index d9351c8b6b8..a5455f4a6d1 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -25,5 +25,40 @@ "param_mappings": { "max_completion_tokens": "max_tokens" } + }, + "synthetic": { + "base_url": "https://api.synthetic.new/openai/v1", + "api_key_env": "SYNTHETIC_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + }, + "apertis": { + "base_url": "https://api.stima.tech/v1", + "api_key_env": "STIMA_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + }, + "nano-gpt": { + "base_url": "https://nano-gpt.com/api/v1", + "api_key_env": "NANOGPT_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + }, + "poe": { + "base_url": "https://api.poe.com/v1", + "api_key_env": "POE_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } + }, + "chutes": { + "base_url": "https://llm.chutes.ai/v1/", + "api_key_env": "CHUTES_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } -} \ No newline at end of file +} diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3416459bc28..2d99a56677b 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3014,6 +3014,12 @@ class LlmProviders(str, Enum): AMAZON_NOVA = "amazon_nova" A2A_AGENT = "a2a_agent" LANGGRAPH = "langgraph" + SYNTHETIC = "synthetic" + APERTIS = "apertis" + NANOGPT = "nano-gpt" + POE = "poe" + CHUTES = "chutes" + # Create a set of all provider values for quick lookup diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 152b3df52e6..82b48b67195 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -137,6 +137,22 @@ "a2a": true } }, + "apertis": { + "display_name": "Apertis (`apertis`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "assemblyai": { "display_name": "AssemblyAI (`assemblyai`)", "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai", @@ -358,6 +374,22 @@ "a2a": true } }, + "chutes": { + "display_name": "Chutes (`chutes`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "clarifai": { "display_name": "Clarifai (`clarifai`)", "url": "https://docs.litellm.ai/docs/providers/clarifai", @@ -1226,6 +1258,22 @@ "a2a": true } }, + "nanogpt": { + "display_name": "NanoGPT (`nanogpt`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "nebius": { "display_name": "Nebius AI Studio (`nebius`)", "url": "https://docs.litellm.ai/docs/providers/nebius", @@ -1507,6 +1555,22 @@ "a2a": true } }, + "poe": { + "display_name": "Poe (`poe`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "publicai": { "display_name": "PublicAI (`publicai`)", "url": "https://docs.litellm.ai/docs/providers/publicai", @@ -1676,6 +1740,22 @@ "a2a": true } }, + "synthetic": { + "display_name": "Synthetic (`synthetic`)", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "text-completion-codestral": { "display_name": "Text Completion Codestral (`text-completion-codestral`)", "url": "https://docs.litellm.ai/docs/providers/codestral", @@ -2068,4 +2148,4 @@ } } } -} \ No newline at end of file +} From c4b2c570f01ca8f5a1aad3160d4efdb03bebc5ce Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 23 Dec 2025 07:19:14 -0300 Subject: [PATCH 018/388] feat(pricing): add Azure gpt-image-1.5 pricing to cost map (#18347) Add missing pricing entries for azure/gpt-image-1.5 and azure/gpt-image-1.5-2025-12-16 to model_prices_and_context_window.json. These models use token-based pricing (same as OpenAI): - Text input: $5.00/1M tokens - Image input: $8.00/1M tokens - Image output: $32.00/1M tokens - Cached text: $1.25/1M tokens - Cached image: $2.00/1M tokens --- model_prices_and_context_window.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 52c86695149..ea7e13c5487 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3707,6 +3707,32 @@ "/v1/images/generations" ] }, + "azure/gpt-image-1.5": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_image_token": 8e-06, + "litellm_provider": "azure", + "mode": "image_generation", + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "azure/gpt-image-1.5-2025-12-16": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_image_token": 8e-06, + "litellm_provider": "azure", + "mode": "image_generation", + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, "azure/low/1024-x-1024/gpt-image-1-mini": { "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", From 40fac60ea719e1fb0a67b64de2d571b7904c2220 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 23 Dec 2025 07:20:57 -0300 Subject: [PATCH 019/388] docs(openai): fix gpt-5-mini reasoning_effort supported values (#18346) Remove 'none' from gpt-5-mini's supported reasoning_effort values in the documentation table. gpt-5-mini does not support reasoning_effort="none", only minimal, low, medium, and high. --- docs/my-website/docs/providers/openai.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 509a106d8a4..80645a51ac5 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -495,7 +495,7 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ |-------|----------------------|------------------| | `gpt-5.1` | `none` | `none`, `low`, `medium`, `high` | | `gpt-5` | `medium` | `minimal`, `low`, `medium`, `high` | -| `gpt-5-mini` | `medium` | `none`, `minimal`, `low`, `medium`, `high` | +| `gpt-5-mini` | `medium` | `minimal`, `low`, `medium`, `high` | | `gpt-5-nano` | `none` | `none`, `low`, `medium`, `high` | | `gpt-5-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | | `gpt-5.1-codex` | `adaptive` | `low`, `medium`, `high` (no `minimal`) | From 73b64e53ff9020a7c56a19785367c5b27f3ca05e Mon Sep 17 00:00:00 2001 From: Emin Askerov Date: Tue, 23 Dec 2025 11:21:57 +0100 Subject: [PATCH 020/388] Add azure_ai/gpt-oss-120b model pricing details (#18317) Added pricing and configuration details for the azure_ai/gpt-oss-120b model, including costs and capabilities. --- model_prices_and_context_window.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ea7e13c5487..d72dd9a6d26 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1357,6 +1357,20 @@ "litellm_provider": "azure", "mode": "chat" }, + "azure_ai/gpt-oss-120b": { + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "litellm_provider": "azure_ai", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, From ca28ab8a0ab48e9336ba6614e70c30cbd4456393 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:08:52 +0530 Subject: [PATCH 021/388] Add: transformation file for minmax anthropic endpoint --- .../llms/minimax/messages/transformation.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 litellm/llms/minimax/messages/transformation.py diff --git a/litellm/llms/minimax/messages/transformation.py b/litellm/llms/minimax/messages/transformation.py new file mode 100644 index 00000000000..27d28f02d83 --- /dev/null +++ b/litellm/llms/minimax/messages/transformation.py @@ -0,0 +1,81 @@ +""" +MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API +""" +from typing import Optional + +import litellm +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str + + +class MinimaxMessagesConfig(AnthropicMessagesConfig): + """ + MiniMax Anthropic configuration that extends AnthropicConfig. + MiniMax provides an Anthropic-compatible API at: + - International: https://api.minimax.io/anthropic + - China: https://api.minimaxi.com/anthropic + + Supported models: + - MiniMax-M2.1 + - MiniMax-M2.1-lightning + - MiniMax-M2 + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "minimax" + + @staticmethod + def get_api_key(api_key: Optional[str] = None) -> Optional[str]: + """ + Get MiniMax API key from environment or parameters. + """ + return ( + api_key + or get_secret_str("MINIMAX_API_KEY") + or litellm.api_key + ) + + @staticmethod + def get_api_base( + api_base: Optional[str] = None, + ) -> str: + """ + Get MiniMax API base URL. + Defaults to international endpoint: https://api.minimax.io/anthropic + For China, set to: https://api.minimaxi.com/anthropic + """ + return ( + api_base + or get_secret_str("MINIMAX_API_BASE") + or "https://api.minimax.io/anthropic/v1/messages" + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for MiniMax API. + Override to ensure we use MiniMax's endpoint, not Anthropic's. + """ + # Get the base URL (either provided or default MiniMax endpoint) + base_url = self.get_api_base(api_base=api_base) + + # If the base URL already includes the full path, return it + if base_url.endswith("/v1/messages"): + return base_url + + # Otherwise append the messages endpoint + if base_url.endswith("/"): + return f"{base_url}v1/messages" + else: + return f"{base_url}/v1/messages" + From 4bae4d00f5cf0fc480218040d358987e97058783 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:09:35 +0530 Subject: [PATCH 022/388] Add: transformation file for minmax anthropic endpoint --- litellm/litellm_core_utils/get_llm_provider_logic.py | 5 ++++- litellm/types/utils.py | 1 + litellm/utils.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a23fce891b9..a707bdedad3 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -4,8 +4,8 @@ import httpx import litellm from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH -from litellm.secret_managers.main import get_secret, get_secret_str from litellm.llms.openai_like.json_loader import JSONProviderRegistry +from litellm.secret_managers.main import get_secret, get_secret_str from ..types.router import LiteLLM_Params @@ -267,6 +267,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "api.moonshot.ai/v1": custom_llm_provider = "moonshot" dynamic_api_key = get_secret_str("MOONSHOT_API_KEY") + elif endpoint == "api.minimax.io/anthropic" or endpoint == "api.minimaxi.com/anthropic": + custom_llm_provider = "minimax" + dynamic_api_key = get_secret_str("MINIMAX_API_KEY") elif endpoint == "platform.publicai.co/v1": custom_llm_provider = "publicai" dynamic_api_key = get_secret_str("PUBLICAI_API_KEY") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3416459bc28..47d9ece5e37 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3014,6 +3014,7 @@ class LlmProviders(str, Enum): AMAZON_NOVA = "amazon_nova" A2A_AGENT = "a2a_agent" LANGGRAPH = "langgraph" + MINIMAX = "minimax" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index 805fbafcfce..c0e4dd8c052 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7501,6 +7501,12 @@ class ProviderConfigManager: ) return AzureAnthropicMessagesConfig() + elif litellm.LlmProviders.MINIMAX == provider: + from litellm.llms.minimax.messages.transformation import ( + MinimaxMessagesConfig, + ) + + return MinimaxMessagesConfig() return None @staticmethod From 743960ad0a95b98f0da2642ce35184014f84e87a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:10:06 +0530 Subject: [PATCH 023/388] Add pricing for minmax models in model map --- ...odel_prices_and_context_window_backup.json | 42 +++++++++++++++++++ model_prices_and_context_window.json | 42 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f4b42d1fd6e..9a938164f3d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -19580,6 +19580,48 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax/MiniMax-M2.1": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.1-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "max_input_tokens": 200000, + "max_output_tokens": 8192 + }, "mistral.magistral-small-2509": { "input_cost_per_token": 5e-07, "litellm_provider": "bedrock_converse", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f4b42d1fd6e..9a938164f3d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19580,6 +19580,48 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax/MiniMax-M2.1": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2.1-lightning": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "max_input_tokens": 1000000, + "max_output_tokens": 8192 + }, + "minimax/MiniMax-M2": { + "input_cost_per_token": 3e-07, + "output_cost_per_token": 1.2e-06, + "cache_read_input_token_cost": 3e-08, + "cache_creation_input_token_cost": 3.75e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_system_messages": true, + "max_input_tokens": 200000, + "max_output_tokens": 8192 + }, "mistral.magistral-small-2509": { "input_cost_per_token": 5e-07, "litellm_provider": "bedrock_converse", From 2f9042c9748ae6d3976d4e48bc46b6d59aba8cd6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:10:36 +0530 Subject: [PATCH 024/388] Add tests related to minmax doc --- tests/test_litellm/llms/minimax/__init__.py | 2 + .../llms/minimax/messages/__init__.py | 2 + .../minimax/messages/test_transformation.py | 147 ++++++++++++++++++ 3 files changed, 151 insertions(+) create mode 100644 tests/test_litellm/llms/minimax/__init__.py create mode 100644 tests/test_litellm/llms/minimax/messages/__init__.py create mode 100644 tests/test_litellm/llms/minimax/messages/test_transformation.py diff --git a/tests/test_litellm/llms/minimax/__init__.py b/tests/test_litellm/llms/minimax/__init__.py new file mode 100644 index 00000000000..19c644e5d98 --- /dev/null +++ b/tests/test_litellm/llms/minimax/__init__.py @@ -0,0 +1,2 @@ +# MiniMax tests + diff --git a/tests/test_litellm/llms/minimax/messages/__init__.py b/tests/test_litellm/llms/minimax/messages/__init__.py new file mode 100644 index 00000000000..8672b141150 --- /dev/null +++ b/tests/test_litellm/llms/minimax/messages/__init__.py @@ -0,0 +1,2 @@ +# MiniMax messages tests + diff --git a/tests/test_litellm/llms/minimax/messages/test_transformation.py b/tests/test_litellm/llms/minimax/messages/test_transformation.py new file mode 100644 index 00000000000..bbb30b652af --- /dev/null +++ b/tests/test_litellm/llms/minimax/messages/test_transformation.py @@ -0,0 +1,147 @@ +""" +Test MiniMax Anthropic-compatible API support +""" +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../") +) # Adds the parent directory to the system path + +import litellm +from litellm import completion +from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig + + +def test_minimax_anthropic_config(): + """Test that MinimaxMessagesConfig is properly configured""" + config = MinimaxMessagesConfig() + + # Test custom_llm_provider + assert config.custom_llm_provider == "minimax" + + # Test get_api_base default + api_base = config.get_api_base() + assert api_base == "https://api.minimax.io/anthropic/v1/messages" + + # Test get_api_base with custom value + custom_base = config.get_api_base(api_base="https://api.minimaxi.com/anthropic/v1/messages") + assert custom_base == "https://api.minimaxi.com/anthropic/v1/messages" + + +def test_minimax_provider_routing(): + """Test that minimax provider is properly routed""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + # Test with minimax/ prefix + model, provider, api_key, api_base = get_llm_provider( + model="minimax/MiniMax-M2.1", + api_base="https://api.minimax.io/anthropic/v1/messages" + ) + assert provider == "minimax" + assert model == "MiniMax-M2.1" + + +def test_minimax_provider_config_manager(): + """Test that ProviderConfigManager returns MinimaxMessagesConfig""" + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_anthropic_messages_config( + model="MiniMax-M2.1", + provider=LlmProviders.MINIMAX + ) + + assert config is not None + assert isinstance(config, MinimaxMessagesConfig) + assert config.custom_llm_provider == "minimax" + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_completion_basic(): + """Test basic completion with MiniMax Anthropic-compatible API""" + response = completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/anthropic/v1/messages" + ) + + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_completion_with_thinking(): + """Test completion with thinking parameter (MiniMax M2.1 feature)""" + response = completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Solve this problem: 2+2=?"}], + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/anthropic/v1/messages", + thinking={"type": "enabled", "budget_tokens": 1000} + ) + + assert response is not None + # Check if thinking content is present in response + for choice in response.choices: + if hasattr(choice.message, "content"): + # MiniMax returns thinking blocks similar to Anthropic + assert choice.message.content is not None + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_completion_with_tools(): + """Test completion with tool calling (function calling)""" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + } + }, + "required": ["location"], + }, + }, + } + ] + + response = completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "What's the weather in San Francisco?"}], + tools=tools, + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/anthropic/v1/messages" + ) + + assert response is not None + assert hasattr(response, "choices") + + +if __name__ == "__main__": + # Run basic tests that don't require API key + print("Testing MiniMax Anthropic Config...") + test_minimax_anthropic_config() + print("✓ Config test passed") + + print("\nTesting MiniMax Provider Routing...") + test_minimax_provider_routing() + print("✓ Routing test passed") + + print("\nTesting MiniMax Provider Config Manager...") + test_minimax_provider_config_manager() + print("✓ Provider config manager test passed") + + print("\n✅ All basic tests passed!") + From 403875256c603dc71755829d8b20c0d56f54d435 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:11:26 +0530 Subject: [PATCH 025/388] Add minmax documentation --- docs/my-website/docs/providers/minmax.md | 192 +++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 193 insertions(+) create mode 100644 docs/my-website/docs/providers/minmax.md diff --git a/docs/my-website/docs/providers/minmax.md b/docs/my-website/docs/providers/minmax.md new file mode 100644 index 00000000000..0e8f18d2505 --- /dev/null +++ b/docs/my-website/docs/providers/minmax.md @@ -0,0 +1,192 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# MiniMax - v1/messages + +## Overview + +Litellm provides anthropic specs compatible support for minmax + +## Supported Models + +MiniMax offers three models through their Anthropic-compatible API: + +| Model | Description | Input Cost | Output Cost | Prompt Caching Read | Prompt Caching Write | +|-------|-------------|------------|-------------|---------------------|----------------------| +| **MiniMax-M2.1** | Powerful Multi-Language Programming with Enhanced Programming Experience (~60 tps) | $0.3/M tokens | $1.2/M tokens | $0.03/M tokens | $0.375/M tokens | +| **MiniMax-M2.1-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | $0.03/M tokens | $0.375/M tokens | +| **MiniMax-M2** | Agentic capabilities, Advanced reasoning | $0.3/M tokens | $1.2/M tokens | $0.03/M tokens | $0.375/M tokens | + + +## Usage Examples + +### Basic Chat Completion + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/anthropic/v1/messages" +) + +print(response.choices[0].message.content) +``` + +### Using Environment Variables + +```bash +export MINIMAX_API_KEY="your-minimax-api-key" +export MINIMAX_API_BASE="https://api.minimax.io/anthropic/v1/messages" +``` + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +### With Thinking (M2.1 Feature) + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Solve: 2+2=?"}], + thinking={"type": "enabled", "budget_tokens": 1000}, + api_key="your-minimax-api-key" +) + +# Access thinking content +for block in response.choices[0].message.content: + if hasattr(block, 'type') and block.type == 'thinking': + print(f"Thinking: {block.thinking}") +``` + +### With Tool Calling + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } +] + +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "What's the weather in SF?"}], + tools=tools, + api_key="your-minimax-api-key" +) +``` + +### Streaming + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True, + api_key="your-minimax-api-key" +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +## Usage with LiteLLM Proxy + +You can use MiniMax models with the Anthropic SDK by routing through LiteLLM Proxy: + +| Step | Description | +|------|-------------| +| **1. Start LiteLLM Proxy** | Configure proxy with MiniMax models in `config.yaml` | +| **2. Set Environment Variables** | Point Anthropic SDK to proxy endpoint | +| **3. Use Anthropic SDK** | Call MiniMax models using native Anthropic SDK | + +### Step 1: Configure LiteLLM Proxy + +Create a `config.yaml`: + +```yaml +model_list: + - model_name: minimax/MiniMax-M2.1 + litellm_params: + model: minimax/MiniMax-M2.1 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/anthropic/v1/messages +``` + +Start the proxy: + +```bash +litellm --config config.yaml +``` + +### Step 2: Use with Anthropic SDK + +```python +import os +os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000" +os.environ["ANTHROPIC_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +import anthropic + +client = anthropic.Anthropic() + +message = client.messages.create( + model="minimax/MiniMax-M2.1", + max_tokens=1000, + system="You are a helpful assistant.", + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hi, how are you?" + } + ] + } + ] +) + +for block in message.content: + if block.type == "thinking": + print(f"Thinking:\n{block.thinking}\n") + elif block.type == "text": + print(f"Text:\n{block.text}\n") +``` +## Cost Calculation + +Cost calculation works automatically using the pricing information in `model_prices_and_context_window.json`. + +Example: +```python +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-minimax-api-key" +) + +# Access cost information +print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") +``` + + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b6b8fe1223d..9801c764acc 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -722,6 +722,7 @@ const sidebars = { "providers/meta_llama", "providers/milvus_vector_stores", "providers/mistral", + "providers/minimax", "providers/moonshot", "providers/morph", "providers/nebius", From 0174c56c907c722879842b869de936832ff65fa4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:23:27 +0530 Subject: [PATCH 026/388] Fix: documentation for litellm sdk --- docs/my-website/docs/providers/minmax.md | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/docs/my-website/docs/providers/minmax.md b/docs/my-website/docs/providers/minmax.md index 0e8f18d2505..7019b8a59f5 100644 --- a/docs/my-website/docs/providers/minmax.md +++ b/docs/my-website/docs/providers/minmax.md @@ -25,7 +25,7 @@ MiniMax offers three models through their Anthropic-compatible API: ```python import litellm -response = litellm.completion( +response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", messages=[{"role": "user", "content": "Hello, how are you?"}], api_key="your-minimax-api-key", @@ -45,7 +45,7 @@ export MINIMAX_API_BASE="https://api.minimax.io/anthropic/v1/messages" ```python import litellm -response = litellm.completion( +response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", messages=[{"role": "user", "content": "Hello!"}] ) @@ -54,7 +54,7 @@ response = litellm.completion( ### With Thinking (M2.1 Feature) ```python -response = litellm.completion( +response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", messages=[{"role": "user", "content": "Solve: 2+2=?"}], thinking={"type": "enabled", "budget_tokens": 1000}, @@ -87,7 +87,7 @@ tools = [ } ] -response = litellm.completion( +response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", messages=[{"role": "user", "content": "What's the weather in SF?"}], tools=tools, @@ -95,20 +95,7 @@ response = litellm.completion( ) ``` -### Streaming -```python -response = litellm.completion( - model="minimax/MiniMax-M2.1", - messages=[{"role": "user", "content": "Tell me a story"}], - stream=True, - api_key="your-minimax-api-key" -) - -for chunk in response: - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="") -``` ## Usage with LiteLLM Proxy From af8483b37ebd08e8765fefe055fa12330a038479 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 16:27:35 +0530 Subject: [PATCH 027/388] Fix: documentation for litellm sdk --- docs/my-website/docs/providers/minmax.md | 9 +++-- ...odel_prices_and_context_window_backup.json | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/minmax.md b/docs/my-website/docs/providers/minmax.md index 7019b8a59f5..3c7db7a9b48 100644 --- a/docs/my-website/docs/providers/minmax.md +++ b/docs/my-website/docs/providers/minmax.md @@ -29,7 +29,8 @@ response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", messages=[{"role": "user", "content": "Hello, how are you?"}], api_key="your-minimax-api-key", - api_base="https://api.minimax.io/anthropic/v1/messages" + api_base="https://api.minimax.io/anthropic/v1/messages", + max_tokens=1000 ) print(response.choices[0].message.content) @@ -47,7 +48,8 @@ import litellm response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", - messages=[{"role": "user", "content": "Hello!"}] + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=1000 ) ``` @@ -91,7 +93,8 @@ response = litellm.anthropic.messages.acreate( model="minimax/MiniMax-M2.1", messages=[{"role": "user", "content": "What's the weather in SF?"}], tools=tools, - api_key="your-minimax-api-key" + api_key="your-minimax-api-key", + max_tokens=1000 ) ``` diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e123ba00081..3fe1b4f6a9e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1357,6 +1357,20 @@ "litellm_provider": "azure", "mode": "chat" }, + "azure_ai/gpt-oss-120b": { + "input_cost_per_token": 1.5e-7, + "output_cost_per_token": 6e-7, + "litellm_provider": "azure_ai", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "mode": "chat", + "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, "azure/eu/gpt-4o-2024-08-06": { "deprecation_date": "2026-02-27", "cache_read_input_token_cost": 1.375e-06, @@ -3707,6 +3721,32 @@ "/v1/images/generations" ] }, + "azure/gpt-image-1.5": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_image_token": 8e-06, + "litellm_provider": "azure", + "mode": "image_generation", + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, + "azure/gpt-image-1.5-2025-12-16": { + "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_token_cost": 1.25e-06, + "input_cost_per_token": 5e-06, + "input_cost_per_image_token": 8e-06, + "litellm_provider": "azure", + "mode": "image_generation", + "output_cost_per_image_token": 3.2e-05, + "supported_endpoints": [ + "/v1/images/generations", + "/v1/images/edits" + ] + }, "azure/low/1024-x-1024/gpt-image-1-mini": { "input_cost_per_pixel": 2.0751953125e-09, "litellm_provider": "azure", From 38cf66df012dbf225b115bc52b1d94e8b6eb901c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 17:07:21 +0530 Subject: [PATCH 028/388] Add: chat completion transformation for minmax --- litellm/llms/minimax/chat/__init__.py | 4 + litellm/llms/minimax/chat/transformation.py | 83 +++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 litellm/llms/minimax/chat/__init__.py create mode 100644 litellm/llms/minimax/chat/transformation.py diff --git a/litellm/llms/minimax/chat/__init__.py b/litellm/llms/minimax/chat/__init__.py new file mode 100644 index 00000000000..45bcfd03b49 --- /dev/null +++ b/litellm/llms/minimax/chat/__init__.py @@ -0,0 +1,4 @@ +""" +MiniMax OpenAI-compatible chat API +""" + diff --git a/litellm/llms/minimax/chat/transformation.py b/litellm/llms/minimax/chat/transformation.py new file mode 100644 index 00000000000..ed80ff8aed1 --- /dev/null +++ b/litellm/llms/minimax/chat/transformation.py @@ -0,0 +1,83 @@ +""" +MiniMax OpenAI transformation config - extends OpenAI chat config for MiniMax's OpenAI-compatible API +""" +from typing import Optional + +import litellm +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.secret_managers.main import get_secret_str + + +class MinimaxChatConfig(OpenAIGPTConfig): + """ + MiniMax OpenAI configuration that extends OpenAIGPTConfig. + MiniMax provides an OpenAI-compatible API at: + - International: https://api.minimax.io/v1 + - China: https://api.minimaxi.com/v1 + + Supported models: + - MiniMax-M2.1 + - MiniMax-M2.1-lightning + - MiniMax-M2 + """ + + @staticmethod + def get_api_key(api_key: Optional[str] = None) -> Optional[str]: + """ + Get MiniMax API key from environment or parameters. + """ + return ( + api_key + or get_secret_str("MINIMAX_API_KEY") + or litellm.api_key + ) + + @staticmethod + def get_api_base( + api_base: Optional[str] = None, + ) -> str: + """ + Get MiniMax API base URL. + Defaults to international endpoint: https://api.minimax.io/v1 + For China, set to: https://api.minimaxi.com/v1 + """ + return ( + api_base + or get_secret_str("MINIMAX_API_BASE") + or "https://api.minimax.io/v1" + ) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for MiniMax OpenAI API. + Override to ensure we use MiniMax's endpoint. + """ + # Get the base URL (either provided or default MiniMax endpoint) + base_url = self.get_api_base(api_base=api_base) + + # Ensure it ends with /chat/completions + if base_url.endswith("/chat/completions"): + return base_url + elif base_url.endswith("/v1"): + return f"{base_url}/chat/completions" + elif base_url.endswith("/"): + return f"{base_url}v1/chat/completions" + else: + return f"{base_url}/v1/chat/completions" + + def get_supported_openai_params(self, model: str) -> list: + """ + Get supported OpenAI parameters for MiniMax. + Adds reasoning_split to the list of supported params. + """ + base_params = super().get_supported_openai_params(model=model) + return base_params + ["reasoning_split"] + From b1b8d19d97127a3f238b105a78be0c32e7612d5f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 17:08:44 +0530 Subject: [PATCH 029/388] Add: chat completion transformation for minmax --- litellm/__init__.py | 1 + litellm/_lazy_imports.py | 9 +++++ .../get_llm_provider_logic.py | 3 ++ litellm/main.py | 38 ++++++++++++++++++- litellm/utils.py | 2 + 5 files changed, 52 insertions(+), 1 deletion(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index b20b3c5f8e1..dfcee0e2d3c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1482,6 +1482,7 @@ if TYPE_CHECKING: from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig + from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 6f96f9f8ff3..044fad924ac 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -165,6 +165,7 @@ LLM_CONFIG_NAMES = ( "BytezChatConfig", "CompactifAIChatConfig", "EmpowerChatConfig", + "MinimaxChatConfig", "AiohttpOpenAIChatConfig", "HuggingFaceChatConfig", "HuggingFaceEmbeddingConfig", @@ -750,6 +751,14 @@ def _lazy_import_llm_configs(name: str) -> Any: # noqa: PLR0915 _globals["EmpowerChatConfig"] = _EmpowerChatConfig return _EmpowerChatConfig + if name == "MinimaxChatConfig": + from .llms.minimax.chat.transformation import ( + MinimaxChatConfig as _MinimaxChatConfig, + ) + + _globals["MinimaxChatConfig"] = _MinimaxChatConfig + return _MinimaxChatConfig + if name == "AiohttpOpenAIChatConfig": from .llms.aiohttp_openai.chat.transformation import ( AiohttpOpenAIChatConfig as _AiohttpOpenAIChatConfig, diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 3b0a3a5e1a5..164e2a73e65 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -270,6 +270,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "api.minimax.io/anthropic" or endpoint == "api.minimaxi.com/anthropic": custom_llm_provider = "minimax" dynamic_api_key = get_secret_str("MINIMAX_API_KEY") + elif endpoint == "api.minimax.io/v1" or endpoint == "api.minimaxi.com/v1": + custom_llm_provider = "minimax" + dynamic_api_key = get_secret_str("MINIMAX_API_KEY") elif endpoint == "platform.publicai.co/v1": custom_llm_provider = "publicai" dynamic_api_key = get_secret_str("PUBLICAI_API_KEY") diff --git a/litellm/main.py b/litellm/main.py index 60fe3eb2dec..fe2c2f333fc 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -68,7 +68,6 @@ from litellm.constants import ( DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT, ) from litellm.exceptions import LiteLLMUnknownProvider -from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.audio_utils.utils import ( @@ -98,6 +97,7 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( VertexAIModelRoute, get_vertex_ai_model_route, @@ -2247,6 +2247,42 @@ def completion( # type: ignore # noqa: PLR0915 logging.post_call( input=messages, api_key=api_key, original_response=response ) + elif custom_llm_provider == "minimax": + api_key = ( + api_key + or get_secret_str("MINIMAX_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("MINIMAX_API_BASE") + or "https://api.minimax.io/v1" + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call( + input=messages, api_key=api_key, original_response=response + ) elif ( model in litellm.open_ai_chat_completion_models or custom_llm_provider == "custom_openai" diff --git a/litellm/utils.py b/litellm/utils.py index c0e4dd8c052..d073c7866c2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7224,6 +7224,8 @@ class ProviderConfigManager: return litellm.IBMWatsonXAIConfig() elif litellm.LlmProviders.EMPOWER == provider: return litellm.EmpowerChatConfig() + elif litellm.LlmProviders.MINIMAX == provider: + return litellm.MinimaxChatConfig() elif litellm.LlmProviders.GITHUB == provider: return litellm.GithubChatConfig() elif litellm.LlmProviders.COMPACTIFAI == provider: From 26c039614648ee278c96c06cf7ecda55dc4c3550 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 17:10:06 +0530 Subject: [PATCH 030/388] Add documentation for chat compeltion minmax --- docs/my-website/docs/providers/minmax.md | 201 +++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/docs/my-website/docs/providers/minmax.md b/docs/my-website/docs/providers/minmax.md index 3c7db7a9b48..b76f1271589 100644 --- a/docs/my-website/docs/providers/minmax.md +++ b/docs/my-website/docs/providers/minmax.md @@ -163,6 +163,207 @@ for block in message.content: elif block.type == "text": print(f"Text:\n{block.text}\n") ``` + +# MiniMax - v1/chat/completions + +## Usage with LiteLLM SDK + +You can use MiniMax's OpenAI-compatible API directly with LiteLLM: + +### Basic Chat Completion + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello, how are you?"} + ], + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +print(response.choices[0].message.content) +``` + +### Using Environment Variables + +```bash +export MINIMAX_API_KEY="your-minimax-api-key" +export MINIMAX_API_BASE="https://api.minimax.io/v1" +``` + +```python +import litellm + +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +### With Reasoning Split + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Solve: 2+2=?"} + ], + extra_body={"reasoning_split": True}, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +# Access reasoning details if available +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking: {response.choices[0].message.reasoning_details}") +print(f"Response: {response.choices[0].message.content}") +``` + +### With Tool Calling + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } +] + +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "What's the weather in SF?"}], + tools=tools, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) +``` + +### Streaming + +```python +response = litellm.completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True, + api_key="your-minimax-api-key", + api_base="https://api.minimax.io/v1" +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + + +## Usage with OpenAI SDK via LiteLLM Proxy + +You can also use MiniMax models with the OpenAI SDK by routing through LiteLLM Proxy: + +| Step | Description | +|------|-------------| +| **1. Start LiteLLM Proxy** | Configure proxy with MiniMax models in `config.yaml` | +| **2. Set Environment Variables** | Point OpenAI SDK to proxy endpoint | +| **3. Use OpenAI SDK** | Call MiniMax models using native OpenAI SDK | + +### Step 1: Configure LiteLLM Proxy + +Create a `config.yaml`: + +```yaml +model_list: + - model_name: minimax/MiniMax-M2.1 + litellm_params: + model: minimax/MiniMax-M2.1 + api_key: os.environ/MINIMAX_API_KEY + api_base: https://api.minimax.io/v1 +``` + +Start the proxy: + +```bash +litellm --config config.yaml +``` + +### Step 2: Use with OpenAI SDK + +```python +import os +os.environ["OPENAI_BASE_URL"] = "http://localhost:4000" +os.environ["OPENAI_API_KEY"] = "sk-1234" # Your LiteLLM proxy key + +from openai import OpenAI + +client = OpenAI() + +response = client.chat.completions.create( + model="minimax/MiniMax-M2.1", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hi, how are you?"}, + ], + # Set reasoning_split=True to separate thinking content + extra_body={"reasoning_split": True}, +) + +# Access thinking and response +if hasattr(response.choices[0].message, 'reasoning_details'): + print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n") +print(f"Text:\n{response.choices[0].message.content}\n") +``` + +### Streaming with OpenAI SDK + +```python +from openai import OpenAI + +client = OpenAI() + +stream = client.chat.completions.create( + model="minimax/MiniMax-M2.1", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Tell me a story"}, + ], + extra_body={"reasoning_split": True}, + stream=True, +) + +reasoning_buffer = "" +text_buffer = "" + +for chunk in stream: + if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details: + for detail in chunk.choices[0].delta.reasoning_details: + if "text" in detail: + reasoning_text = detail["text"] + new_reasoning = reasoning_text[len(reasoning_buffer):] + if new_reasoning: + print(new_reasoning, end="", flush=True) + reasoning_buffer = reasoning_text + + if chunk.choices[0].delta.content: + content_text = chunk.choices[0].delta.content + new_text = content_text[len(text_buffer):] if text_buffer else content_text + if new_text: + print(new_text, end="", flush=True) + text_buffer = content_text +``` + ## Cost Calculation Cost calculation works automatically using the pricing information in `model_prices_and_context_window.json`. From 4e77dc67d231b6693d4de254f5f0b6c80a471d0f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 17:10:46 +0530 Subject: [PATCH 031/388] Add tests for chat completion minmax --- .../llms/minimax/chat/__init__.py | 2 + .../llms/minimax/chat/test_transformation.py | 225 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 tests/test_litellm/llms/minimax/chat/__init__.py create mode 100644 tests/test_litellm/llms/minimax/chat/test_transformation.py diff --git a/tests/test_litellm/llms/minimax/chat/__init__.py b/tests/test_litellm/llms/minimax/chat/__init__.py new file mode 100644 index 00000000000..6c63920b3ea --- /dev/null +++ b/tests/test_litellm/llms/minimax/chat/__init__.py @@ -0,0 +1,2 @@ +# MiniMax chat tests + diff --git a/tests/test_litellm/llms/minimax/chat/test_transformation.py b/tests/test_litellm/llms/minimax/chat/test_transformation.py new file mode 100644 index 00000000000..aa7105077a0 --- /dev/null +++ b/tests/test_litellm/llms/minimax/chat/test_transformation.py @@ -0,0 +1,225 @@ +""" +Test MiniMax OpenAI-compatible API support +""" +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../") +) # Adds the parent directory to the system path + +import litellm +from litellm import completion +from litellm.llms.minimax.chat.transformation import MinimaxChatConfig + + +def test_minimax_chat_config(): + """Test that MinimaxChatConfig is properly configured""" + config = MinimaxChatConfig() + + # Test get_api_base default + api_base = config.get_api_base() + assert api_base == "https://api.minimax.io/v1" + + # Test get_api_base with custom value + custom_base = config.get_api_base(api_base="https://api.minimaxi.com/v1") + assert custom_base == "https://api.minimaxi.com/v1" + + # Test get_complete_url + complete_url = config.get_complete_url( + api_base="https://api.minimax.io/v1", + api_key=None, + model="MiniMax-M2.1", + optional_params={}, + litellm_params={}, + stream=False + ) + assert complete_url == "https://api.minimax.io/v1/chat/completions" + + +def test_minimax_chat_config_url_variations(): + """Test URL handling with different base URL formats""" + config = MinimaxChatConfig() + + # Test with /v1 ending + url1 = config.get_complete_url( + api_base="https://api.minimax.io/v1", + api_key=None, + model="MiniMax-M2.1", + optional_params={}, + litellm_params={}, + ) + assert url1 == "https://api.minimax.io/v1/chat/completions" + + # Test with trailing slash + url2 = config.get_complete_url( + api_base="https://api.minimax.io/", + api_key=None, + model="MiniMax-M2.1", + optional_params={}, + litellm_params={}, + ) + assert url2 == "https://api.minimax.io/v1/chat/completions" + + # Test without trailing slash + url3 = config.get_complete_url( + api_base="https://api.minimax.io", + api_key=None, + model="MiniMax-M2.1", + optional_params={}, + litellm_params={}, + ) + assert url3 == "https://api.minimax.io/v1/chat/completions" + + # Test with full path already + url4 = config.get_complete_url( + api_base="https://api.minimax.io/v1/chat/completions", + api_key=None, + model="MiniMax-M2.1", + optional_params={}, + litellm_params={}, + ) + assert url4 == "https://api.minimax.io/v1/chat/completions" + + +def test_minimax_provider_routing(): + """Test that minimax provider is properly routed""" + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + # Test with minimax/ prefix + model, provider, api_key, api_base = get_llm_provider( + model="minimax/MiniMax-M2.1", + api_base="https://api.minimax.io/v1" + ) + assert provider == "minimax" + assert model == "MiniMax-M2.1" + + +def test_minimax_provider_config_manager(): + """Test that ProviderConfigManager returns MinimaxChatConfig""" + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_chat_config( + model="MiniMax-M2.1", + provider=LlmProviders.MINIMAX + ) + + assert config is not None + assert isinstance(config, MinimaxChatConfig) + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_chat_completion_basic(): + """Test basic chat completion with MiniMax OpenAI-compatible API""" + response = completion( + model="minimax/MiniMax-M2.1", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello, how are you?"} + ], + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/v1" + ) + + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_chat_completion_with_reasoning_split(): + """Test completion with reasoning_split parameter (MiniMax M2.1 feature)""" + response = completion( + model="minimax/MiniMax-M2.1", + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Solve this problem: 2+2=?"} + ], + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/v1", + extra_body={"reasoning_split": True} + ) + + assert response is not None + # Check if reasoning_details is present in response + if hasattr(response.choices[0].message, "reasoning_details"): + assert response.choices[0].message.reasoning_details is not None + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_chat_completion_with_tools(): + """Test completion with tool calling (function calling)""" + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + } + }, + "required": ["location"], + }, + }, + } + ] + + response = completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "What's the weather in San Francisco?"}], + tools=tools, + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/v1" + ) + + assert response is not None + assert hasattr(response, "choices") + + +@pytest.mark.skip(reason="Requires actual MiniMax API key") +def test_minimax_chat_completion_streaming(): + """Test streaming completion""" + response = completion( + model="minimax/MiniMax-M2.1", + messages=[{"role": "user", "content": "Count to 5"}], + stream=True, + api_key=os.getenv("MINIMAX_API_KEY"), + api_base="https://api.minimax.io/v1" + ) + + chunks = [] + for chunk in response: + chunks.append(chunk) + + assert len(chunks) > 0 + + +if __name__ == "__main__": + # Run basic tests that don't require API key + print("Testing MiniMax Chat Config...") + test_minimax_chat_config() + print("✓ Config test passed") + + print("\nTesting MiniMax Chat Config URL Variations...") + test_minimax_chat_config_url_variations() + print("✓ URL variations test passed") + + print("\nTesting MiniMax Provider Routing...") + test_minimax_provider_routing() + print("✓ Routing test passed") + + print("\nTesting MiniMax Provider Config Manager...") + test_minimax_provider_config_manager() + print("✓ Provider config manager test passed") + + print("\n✅ All basic tests passed!") + From 16985ab2f3ea021383cbee4e0e64ae8e9c3be2a1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 17:13:48 +0530 Subject: [PATCH 032/388] Potential fix for code scanning alert no. 3942: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 7a1b0a34361..2ee2296ab6e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -915,7 +915,7 @@ def function_setup( # noqa: PLR0915 # Only process if target is NOT a Gemini model if not _is_gemini_model(model, custom_llm_provider): verbose_logger.debug( - f"Removing thought signatures from tool call IDs for non-Gemini model: {model}" + "Removing thought signatures from tool call IDs for non-Gemini model" ) # Process messages to remove thought signatures From dce95d3a03cedc562bce182343b9dfb818fb03d0 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 17:16:11 +0530 Subject: [PATCH 033/388] fix mypy issue --- litellm/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 2ee2296ab6e..9216e1e53e0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -927,9 +927,9 @@ def function_setup( # noqa: PLR0915 if "messages" in kwargs: kwargs["messages"] = processed_messages elif len(args) > 1: - args = list(args) - args[1] = processed_messages - args = tuple(args) + args_list = list(args) + args_list[1] = processed_messages + args = tuple(args_list) except Exception as e: # Log the error but don't fail the request From b2b0604173c7e4709da7d3fdfd64abe7d8deaa61 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 21:36:21 +0530 Subject: [PATCH 034/388] Fix : mypy error --- .../test_thought_signature_in_tool_call_id.py | 276 +++++++----------- 1 file changed, 105 insertions(+), 171 deletions(-) diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 46bb8930a7a..5fe51ed23b9 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -10,15 +10,16 @@ enable_preview_features=True to be enabled. """ import pytest + import litellm -from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, -) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, - convert_to_gemini_tool_call_invoke, _encode_tool_call_id_with_signature, _get_thought_signature_from_tool, + convert_to_gemini_tool_call_invoke, +) +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, ) from litellm.types.llms.vertex_ai import HttpxPartType @@ -71,52 +72,36 @@ def test_tool_call_id_includes_signature_in_response(enable_preview_features): """Test that tool call IDs in responses include embedded thought signatures only when preview features are enabled""" test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - # Save original state - original_flag = litellm.enable_preview_features - litellm.enable_preview_features = enable_preview_features - - try: - parts_with_signature = [ - HttpxPartType( - functionCall={ - "name": "get_current_temperature", - "args": {"location": "Paris"}, - }, - thoughtSignature=test_signature, - ) - ] - - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=parts_with_signature, - cumulative_tool_call_idx=0, - is_function_call=False, + parts_with_signature = [ + HttpxPartType( + functionCall={ + "name": "get_current_temperature", + "args": {"location": "Paris"}, + }, + thoughtSignature=test_signature, ) + ] - # Verify tool call exists - assert tools is not None - assert len(tools) == 1 - tool_call_id = tools[0]["id"] - - # Verify signature is always in provider_specific_fields - assert tools[0].get("provider_specific_fields", {}).get("thought_signature") == test_signature + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts_with_signature, + cumulative_tool_call_idx=0, + is_function_call=False, + ) - if enable_preview_features: - # When preview features enabled, signature should be embedded in ID - assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id - # Verify we can decode it using the factory function - tool_obj = {"id": tool_call_id, "type": "function"} - decoded_sig = _get_thought_signature_from_tool(tool_obj) - assert decoded_sig == test_signature - else: - # When preview features disabled, signature should NOT be embedded in ID - assert THOUGHT_SIGNATURE_SEPARATOR not in tool_call_id - # But we can still extract from provider_specific_fields - tool_obj = {"id": tool_call_id, "type": "function", "provider_specific_fields": {"thought_signature": test_signature}} - decoded_sig = _get_thought_signature_from_tool(tool_obj) - assert decoded_sig == test_signature - finally: - # Restore original state - litellm.enable_preview_features = original_flag + # Verify tool call exists + assert tools is not None + assert len(tools) == 1 + tool_call_id = tools[0]["id"] + + # Verify signature is always in provider_specific_fields + assert tools[0].get("provider_specific_fields", {}).get("thought_signature") == test_signature + + # When preview features enabled, signature should be embedded in ID + assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id + # Verify we can decode it using the factory function + tool_obj = {"id": tool_call_id, "type": "function"} + decoded_sig = _get_thought_signature_from_tool(tool_obj) + assert decoded_sig == test_signature def test_get_thought_signature_backward_compatibility(): @@ -204,90 +189,57 @@ def test_openai_client_e2e_flow(enable_preview_features): """ test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - # Save original state - original_flag = litellm.enable_preview_features - litellm.enable_preview_features = enable_preview_features + # Step 1: Gemini returns function call with thought signature + gemini_parts = [ + HttpxPartType( + functionCall={ + "name": "get_current_temperature", + "args": {"location": "Paris"}, + }, + thoughtSignature=test_signature, + ) + ] - try: - # Step 1: Gemini returns function call with thought signature - gemini_parts = [ - HttpxPartType( - functionCall={ + # Step 2: LiteLLM transforms to OpenAI format + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=gemini_parts, + cumulative_tool_call_idx=0, + is_function_call=False, + ) + + assert tools is not None + assert len(tools) == 1 + tool_call_id = tools[0]["id"] + + assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id + + # Step 3: OpenAI client sends back assistant message + # For the disabled case, we simulate that the client might have provider_specific_fields + # or we use the embedded ID if preview features were enabled + openai_assistant_message = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": tool_call_id, # Preserved from response (with embedded signature) + "type": "function", + "function": { "name": "get_current_temperature", - "args": {"location": "Paris"}, + "arguments": '{"location": "Paris"}', }, - thoughtSignature=test_signature, - ) - ] - - # Step 2: LiteLLM transforms to OpenAI format - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=gemini_parts, - cumulative_tool_call_idx=0, - is_function_call=False, - ) - - assert tools is not None - assert len(tools) == 1 - tool_call_id = tools[0]["id"] - - if enable_preview_features: - # When preview features enabled, signature should be embedded in ID - assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id - else: - # When preview features disabled, signature should NOT be embedded in ID - assert THOUGHT_SIGNATURE_SEPARATOR not in tool_call_id - - # Step 3: OpenAI client sends back assistant message - # For the disabled case, we simulate that the client might have provider_specific_fields - # or we use the embedded ID if preview features were enabled - if enable_preview_features: - openai_assistant_message = { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": tool_call_id, # Preserved from response (with embedded signature) - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - }, - } - ], - } - else: - # When preview features disabled, simulate that provider_specific_fields might be preserved - # (though in real OpenAI client usage, this might not happen) - # For this test, we'll use provider_specific_fields to show extraction still works - openai_assistant_message = { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": tool_call_id, # ID without embedded signature - "type": "function", - "function": { - "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', - }, - "provider_specific_fields": {"thought_signature": test_signature}, - } - ], } + ], + } + # Step 4: LiteLLM converts back to Gemini format, extracting signature + gemini_parts_converted = convert_to_gemini_tool_call_invoke( + openai_assistant_message + ) - # Step 4: LiteLLM converts back to Gemini format, extracting signature - gemini_parts_converted = convert_to_gemini_tool_call_invoke( - openai_assistant_message - ) + # Verify signature is preserved through the round trip + assert len(gemini_parts_converted) == 1 + assert "thoughtSignature" in gemini_parts_converted[0] + assert gemini_parts_converted[0]["thoughtSignature"] == test_signature - # Verify signature is preserved through the round trip - assert len(gemini_parts_converted) == 1 - assert "thoughtSignature" in gemini_parts_converted[0] - assert gemini_parts_converted[0]["thoughtSignature"] == test_signature - finally: - # Restore original state - litellm.enable_preview_features = original_flag @pytest.mark.parametrize("enable_preview_features", [True, False]) @@ -296,54 +248,36 @@ def test_parallel_tool_calls_with_signatures(enable_preview_features): signature1 = "signature_for_first_call" # Only first call has signature (Gemini behavior for parallel calls) - # Save original state - original_flag = litellm.enable_preview_features - litellm.enable_preview_features = enable_preview_features + gemini_parts = [ + HttpxPartType( + functionCall={"name": "get_temperature", "args": {"location": "Paris"}}, + thoughtSignature=signature1, + ), + HttpxPartType( + functionCall={"name": "get_temperature", "args": {"location": "London"}}, + # No signature for second parallel call + ), + ] - try: - gemini_parts = [ - HttpxPartType( - functionCall={"name": "get_temperature", "args": {"location": "Paris"}}, - thoughtSignature=signature1, - ), - HttpxPartType( - functionCall={"name": "get_temperature", "args": {"location": "London"}}, - # No signature for second parallel call - ), - ] + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=gemini_parts, + cumulative_tool_call_idx=0, + is_function_call=False, + ) - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=gemini_parts, - cumulative_tool_call_idx=0, - is_function_call=False, - ) + assert tools is not None + assert len(tools) == 2 - assert tools is not None - assert len(tools) == 2 + # First tool call should have signature in provider_specific_fields + assert tools[0].get("provider_specific_fields", {}).get("thought_signature") == signature1 + + # When preview features enabled, first tool call has signature in ID + assert THOUGHT_SIGNATURE_SEPARATOR in tools[0]["id"] + sig1 = _get_thought_signature_from_tool({"id": tools[0]["id"], "type": "function"}) + assert sig1 == signature1 - # First tool call should have signature in provider_specific_fields - assert tools[0].get("provider_specific_fields", {}).get("thought_signature") == signature1 - - if enable_preview_features: - # When preview features enabled, first tool call has signature in ID - assert THOUGHT_SIGNATURE_SEPARATOR in tools[0]["id"] - sig1 = _get_thought_signature_from_tool({"id": tools[0]["id"], "type": "function"}) - assert sig1 == signature1 - else: - # When preview features disabled, signature should NOT be in ID - assert THOUGHT_SIGNATURE_SEPARATOR not in tools[0]["id"] - # But we can extract from provider_specific_fields - sig1 = _get_thought_signature_from_tool({ - "id": tools[0]["id"], - "type": "function", - "provider_specific_fields": {"thought_signature": signature1} - }) - assert sig1 == signature1 - # Second tool call has no signature in ID (regardless of flag) - assert THOUGHT_SIGNATURE_SEPARATOR not in tools[1]["id"] - sig2 = _get_thought_signature_from_tool({"id": tools[1]["id"], "type": "function"}) - assert sig2 is None - finally: - # Restore original state - litellm.enable_preview_features = original_flag + # Second tool call has no signature in ID (regardless of flag) + assert THOUGHT_SIGNATURE_SEPARATOR not in tools[1]["id"] + sig2 = _get_thought_signature_from_tool({"id": tools[1]["id"], "type": "function"}) + assert sig2 is None From e18cfc0cf6908014956c5f9f5dcc2fbc2a585042 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 21:42:51 +0530 Subject: [PATCH 035/388] corrected provider name --- docs/my-website/docs/providers/{minmax.md => minimax.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/my-website/docs/providers/{minmax.md => minimax.md} (100%) diff --git a/docs/my-website/docs/providers/minmax.md b/docs/my-website/docs/providers/minimax.md similarity index 100% rename from docs/my-website/docs/providers/minmax.md rename to docs/my-website/docs/providers/minimax.md From a2240775c4a813cf2f4eac23000341f87344d48c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 21:46:34 +0530 Subject: [PATCH 036/388] correct doc --- docs/my-website/docs/providers/minimax.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/docs/providers/minimax.md b/docs/my-website/docs/providers/minimax.md index b76f1271589..250c5159a3d 100644 --- a/docs/my-website/docs/providers/minimax.md +++ b/docs/my-website/docs/providers/minimax.md @@ -1,6 +1,8 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +# MiniMax + # MiniMax - v1/messages ## Overview From c7e036692528449ca2c1d258450edacf5e3d5919 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 22 Dec 2025 15:46:54 +0530 Subject: [PATCH 037/388] Add support for minmax tts --- docs/my-website/docs/providers/minimax_tts.md | 281 ++++++++++++ docs/my-website/docs/text_to_speech.md | 3 +- docs/my-website/sidebars.js | 1 + litellm/llms/minimax/__init__.py | 14 + .../llms/minimax/text_to_speech/__init__.py | 8 + .../minimax/text_to_speech/transformation.py | 421 ++++++++++++++++++ litellm/main.py | 45 ++ ...odel_prices_and_context_window_backup.json | 32 ++ litellm/utils.py | 6 + model_prices_and_context_window.json | 32 ++ tests/llm_translation/test_minimax_tts.py | 371 +++++++++++++++ 11 files changed, 1213 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/providers/minimax_tts.md create mode 100644 litellm/llms/minimax/__init__.py create mode 100644 litellm/llms/minimax/text_to_speech/__init__.py create mode 100644 litellm/llms/minimax/text_to_speech/transformation.py create mode 100644 tests/llm_translation/test_minimax_tts.py diff --git a/docs/my-website/docs/providers/minimax_tts.md b/docs/my-website/docs/providers/minimax_tts.md new file mode 100644 index 00000000000..314d705413b --- /dev/null +++ b/docs/my-website/docs/providers/minimax_tts.md @@ -0,0 +1,281 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# MiniMax - Text-to-Speech + +## Overview + +MiniMax provides high-quality text-to-speech synthesis with support for 40+ languages and ultra-low latency. LiteLLM provides a unified OpenAI-compatible interface for MiniMax TTS. + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Logging | ✅ | Works across all integrations | +| Fallbacks | ✅ | Works between supported models | +| Loadbalancing | ✅ | Works between supported models | +| Guardrails | ✅ | Applies to input text | +| Supported Models | speech-2.6-hd, speech-2.6-turbo, speech-02-hd, speech-02-turbo | | + +## Supported Models + +| Model | Description | +|-------|-------------| +| speech-2.6-hd | Ultra-low latency, intelligence parsing, and enhanced naturalness | +| speech-2.6-turbo | Faster, more affordable, ideal for agents | +| speech-02-hd | Superior rhythm and stability with outstanding replication similarity | +| speech-02-turbo | Superior rhythm and stability with enhanced multilingual capabilities | +| speech-01-hd | Previous generation HD model | +| speech-01-turbo | Previous generation turbo model | + +## Quick Start + +## **LiteLLM Python SDK Usage** + +### Basic Usage + +```python +from pathlib import Path +from litellm import speech +import os + +os.environ["MINIMAX_API_KEY"] = "your-api-key" + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="The quick brown fox jumped over the lazy dogs", +) +response.stream_to_file(speech_file_path) +``` + +### Async Usage + +```python +from litellm import aspeech +from pathlib import Path +import os, asyncio + +os.environ["MINIMAX_API_KEY"] = "your-api-key" + +async def test_async_speech(): + speech_file_path = Path(__file__).parent / "speech.mp3" + response = await aspeech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="The quick brown fox jumped over the lazy dogs", + ) + response.stream_to_file(speech_file_path) + +asyncio.run(test_async_speech()) +``` + +### Voice Selection + +MiniMax supports many voices. LiteLLM provides OpenAI-compatible voice names that map to MiniMax voices: + +```python +from litellm import speech + +# OpenAI-compatible voice names +voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] + +for voice in voices: + response = speech( + model="minimax/speech-2.6-hd", + voice=voice, + input=f"This is the {voice} voice", + ) + response.stream_to_file(f"speech_{voice}.mp3") +``` + +You can also use MiniMax-native voice IDs directly: + +```python +response = speech( + model="minimax/speech-2.6-hd", + voice="male-qn-qingse", # MiniMax native voice ID + input="Using native MiniMax voice ID", +) +``` + +### Custom Parameters + +MiniMax TTS supports additional parameters for fine-tuning audio output: + +```python +from litellm import speech + +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="Custom audio parameters", + speed=1.5, # Speed: 0.5 to 2.0 + response_format="mp3", # Format: mp3, pcm, wav, flac + extra_body={ + "vol": 1.2, # Volume: 0.1 to 10 + "pitch": 2, # Pitch adjustment: -12 to 12 + "sample_rate": 32000, # 16000, 24000, or 32000 + "bitrate": 128000, # For MP3: 64000, 128000, 192000, 256000 + "channel": 1, # 1 for mono, 2 for stereo + } +) +response.stream_to_file("custom_speech.mp3") +``` + +### Response Formats + +```python +from litellm import speech + +# MP3 format (default) +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="MP3 format audio", + response_format="mp3", +) + +# PCM format +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="PCM format audio", + response_format="pcm", +) + +# WAV format +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="WAV format audio", + response_format="wav", +) + +# FLAC format +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="FLAC format audio", + response_format="flac", +) +``` + +## **LiteLLM Proxy Usage** + +LiteLLM provides an OpenAI-compatible `/audio/speech` endpoint for MiniMax TTS. + +### Setup + +Add MiniMax to your proxy configuration: + +```yaml +model_list: + - model_name: tts + litellm_params: + model: minimax/speech-2.6-hd + api_key: os.environ/MINIMAX_API_KEY + + - model_name: tts-turbo + litellm_params: + model: minimax/speech-2.6-turbo + api_key: os.environ/MINIMAX_API_KEY +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### Making Requests + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy" + }' \ + --output speech.mp3 +``` + +With custom parameters: + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts", + "input": "Custom parameters example.", + "voice": "nova", + "speed": 1.5, + "response_format": "mp3", + "extra_body": { + "vol": 1.2, + "pitch": 1, + "sample_rate": 32000 + } + }' \ + --output custom_speech.mp3 +``` + +## Voice Mappings + +LiteLLM maps OpenAI-compatible voice names to MiniMax voice IDs: + +| OpenAI Voice | MiniMax Voice ID | Description | +|--------------|------------------|-------------| +| alloy | male-qn-qingse | Male voice | +| echo | male-qn-jingying | Male voice | +| fable | female-shaonv | Female voice | +| onyx | male-qn-badao | Male voice | +| nova | female-yujie | Female voice | +| shimmer | female-tianmei | Female voice | + +You can also use any MiniMax-native voice ID directly by passing it as the `voice` parameter. + + +### Streaming (WebSocket) + +:::note +The current implementation uses MiniMax's HTTP endpoint. For WebSocket streaming support, please refer to MiniMax's official documentation at [https://platform.minimax.io/docs](https://platform.minimax.io/docs). +::: + +## Error Handling + +```python +from litellm import speech +import litellm + +try: + response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="Test input", + ) + response.stream_to_file("output.mp3") +except litellm.exceptions.BadRequestError as e: + print(f"Bad request: {e}") +except litellm.exceptions.AuthenticationError as e: + print(f"Authentication failed: {e}") +except Exception as e: + print(f"Error: {e}") +``` + +### Extra Body Parameters + +Pass these via `extra_body`: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| vol | float | Volume (0.1 to 10) | 1.0 | +| pitch | int | Pitch adjustment (-12 to 12) | 0 | +| sample_rate | int | Sample rate: 16000, 24000, 32000 | 32000 | +| bitrate | int | Bitrate for MP3: 64000, 128000, 192000, 256000 | 128000 | +| channel | int | Audio channels: 1 (mono) or 2 (stereo) | 1 | +| output_format | string | Output format: "hex" or "url" (url returns a URL valid for 24 hours) | hex | diff --git a/docs/my-website/docs/text_to_speech.md b/docs/my-website/docs/text_to_speech.md index ce298b538df..f5788630949 100644 --- a/docs/my-website/docs/text_to_speech.md +++ b/docs/my-website/docs/text_to_speech.md @@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem'; | Fallbacks | ✅ | Works between supported models | | Loadbalancing | ✅ | Works between supported models | | Guardrails | ✅ | Applies to input text (non-streaming only) | -| Supported Providers | OpenAI, Azure OpenAI, Vertex AI, AWS Polly, ElevenLabs | | +| Supported Providers | OpenAI, Azure OpenAI, Vertex AI, AWS Polly, ElevenLabs , MiniMax| | ## **LiteLLM Python SDK Usage** ### Quick Start @@ -105,6 +105,7 @@ litellm --config /path/to/config.yaml | Vertex AI | [Usage](../docs/providers/vertex#text-to-speech-apis) | | Gemini | [Usage](#gemini-text-to-speech) | | ElevenLabs | [Usage](../docs/providers/elevenlabs#text-to-speech-tts) | +| MiniMax | [Usage](../docs/providers/minimax_tts) | ## `/audio/speech` to `/chat/completions` Bridge diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 9801c764acc..ae63bade025 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -723,6 +723,7 @@ const sidebars = { "providers/milvus_vector_stores", "providers/mistral", "providers/minimax", + "providers/minimax_tts", "providers/moonshot", "providers/morph", "providers/nebius", diff --git a/litellm/llms/minimax/__init__.py b/litellm/llms/minimax/__init__.py new file mode 100644 index 00000000000..19093c2dadb --- /dev/null +++ b/litellm/llms/minimax/__init__.py @@ -0,0 +1,14 @@ +""" +MiniMax LLM Provider +""" + +from .text_to_speech.transformation import ( + MinimaxException, + MinimaxTextToSpeechConfig, +) + +__all__ = [ + "MinimaxTextToSpeechConfig", + "MinimaxException", +] + diff --git a/litellm/llms/minimax/text_to_speech/__init__.py b/litellm/llms/minimax/text_to_speech/__init__.py new file mode 100644 index 00000000000..e3fcddeb05f --- /dev/null +++ b/litellm/llms/minimax/text_to_speech/__init__.py @@ -0,0 +1,8 @@ +""" +MiniMax Text-to-Speech module +""" + +from .transformation import MinimaxException, MinimaxTextToSpeechConfig + +__all__ = ["MinimaxTextToSpeechConfig", "MinimaxException"] + diff --git a/litellm/llms/minimax/text_to_speech/transformation.py b/litellm/llms/minimax/text_to_speech/transformation.py new file mode 100644 index 00000000000..a3a75d220ff --- /dev/null +++ b/litellm/llms/minimax/text_to_speech/transformation.py @@ -0,0 +1,421 @@ +""" +MiniMax Text-to-Speech transformation + +Maps OpenAI TTS spec to MiniMax TTS API (WebSocket-based HTTP API) +Reference: https://platform.minimax.io/docs +""" + +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union + +import httpx +from httpx import Headers + +import litellm +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.secret_managers.main import get_secret_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent +else: + LiteLLMLoggingObj = Any + HttpxBinaryResponseContent = Any + + +class MinimaxException(BaseLLMException): + """Custom exception for MiniMax API errors""" + + def __init__( + self, + status_code: int, + message: str, + headers: Optional[Union[dict, Headers]] = None, + ): + super().__init__(status_code=status_code, message=message, headers=headers) + + +class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig): + """ + Configuration for MiniMax Text-to-Speech + + Reference: https://platform.minimax.io/docs + + MiniMax TTS API supports both WebSocket and HTTP endpoints. + This implementation uses the HTTP endpoint for simplicity. + """ + + TTS_BASE_URL = "https://api.minimax.io" + TTS_ENDPOINT_PATH = "/v1/t2a_v2" + + # Voice mappings from OpenAI-style voices to MiniMax voice IDs + # MiniMax supports many voices, these are common mappings + VOICE_MAPPINGS = { + "alloy": "male-qn-qingse", + "echo": "male-qn-jingying", + "fable": "female-shaonv", + "onyx": "male-qn-badao", + "nova": "female-yujie", + "shimmer": "female-tianmei", + } + + # Response format mappings from OpenAI to MiniMax + FORMAT_MAPPINGS = { + "mp3": "mp3", + "pcm": "pcm", + "wav": "wav", + "flac": "flac", + } + + def get_supported_openai_params(self, model: str) -> list: + """ + MiniMax TTS supports these OpenAI parameters + """ + return ["voice", "response_format", "speed"] + + def _extract_voice_id(self, voice: str) -> str: + """ + Normalize the provided voice information into a MiniMax voice_id. + """ + normalized_voice = voice.strip() + mapped_voice = self.VOICE_MAPPINGS.get(normalized_voice.lower()) + return mapped_voice or normalized_voice + + def _resolve_voice_id( + self, + voice: Optional[Union[str, Dict[str, Any]]], + params: Dict[str, Any], + ) -> str: + """ + Determine the MiniMax voice_id based on provided voice input or parameters. + """ + mapped_voice: Optional[str] = None + + if isinstance(voice, str) and voice.strip(): + mapped_voice = self._extract_voice_id(voice) + elif isinstance(voice, dict): + for key in ("voice_id", "id", "name"): + candidate = voice.get(key) + if isinstance(candidate, str) and candidate.strip(): + mapped_voice = self._extract_voice_id(candidate) + break + elif voice is not None: + mapped_voice = self._extract_voice_id(str(voice)) + + if mapped_voice is None: + voice_override = params.pop("voice_id", None) + if isinstance(voice_override, str) and voice_override.strip(): + mapped_voice = self._extract_voice_id(voice_override) + + if mapped_voice is None: + # Default to a common voice if not specified + mapped_voice = "male-qn-qingse" + + return mapped_voice + + def map_openai_params( + self, + model: str, + optional_params: Dict, + voice: Optional[Union[str, Dict]] = None, + drop_params: bool = False, + kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[Optional[str], Dict]: + """ + Map OpenAI parameters to MiniMax TTS parameters + """ + mapped_params: Dict[str, Any] = {} + + # Work on a copy so we don't mutate the caller's dictionary + params = dict(optional_params) if optional_params else {} + + # Extract voice identifier + mapped_voice = self._resolve_voice_id(voice, params) + + # Response/output format + response_format = params.pop("response_format", None) + if isinstance(response_format, str): + mapped_format = self.FORMAT_MAPPINGS.get(response_format, "mp3") + mapped_params["format"] = mapped_format + else: + mapped_params["format"] = "mp3" # Default format + + # Speed parameter (MiniMax supports speed from 0.5 to 2.0) + speed = params.pop("speed", None) + if speed is not None: + try: + speed_value = float(speed) + # Clamp speed to MiniMax's supported range + speed_value = max(0.5, min(2.0, speed_value)) + mapped_params["speed"] = speed_value + except (TypeError, ValueError): + mapped_params["speed"] = 1.0 + else: + mapped_params["speed"] = 1.0 + + # Instructions parameter is OpenAI-specific; omit to prevent API errors + params.pop("instructions", None) + + # Store voice_id for later use in request construction + mapped_params["voice_id"] = mapped_voice + + # Handle extra_body for additional MiniMax-specific parameters + extra_body = params.pop("extra_body", None) + if isinstance(extra_body, dict): + for key, value in extra_body.items(): + if value is not None: + mapped_params[key] = value + + # Pass through any remaining parameters + for key, value in params.items(): + if value is not None: + mapped_params[key] = value + + return mapped_voice, mapped_params + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate MiniMax environment and set up authentication headers + """ + api_key = ( + api_key + or litellm.api_key + or get_secret_str("MINIMAX_API_KEY") + ) + + if api_key is None: + raise ValueError( + "MiniMax API key is required. Set MINIMAX_API_KEY environment variable or pass api_key parameter." + ) + + headers.update( + { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + ) + + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, Headers] + ) -> BaseLLMException: + return MinimaxException( + message=error_message, status_code=status_code, headers=headers + ) + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: Optional[str], + optional_params: Dict, + litellm_params: Dict, + headers: dict, + ) -> TextToSpeechRequestData: + """ + Build the MiniMax TTS request payload. + + MiniMax uses a different structure than OpenAI: + - model: The TTS model to use + - text: The input text + - voice_setting: Voice configuration + - audio_setting: Audio output configuration + """ + params = dict(optional_params) if optional_params else {} + + # Extract parameters + voice_id = params.pop("voice_id", voice or "male-qn-qingse") + speed = params.pop("speed", 1.0) + audio_format = params.pop("format", "mp3") + + # Extract additional voice settings + vol = params.pop("vol", 1.0) # Volume (0.1 to 10) + pitch = params.pop("pitch", 0) # Pitch adjustment (-12 to 12) + + # Extract audio settings + sample_rate = params.pop("sample_rate", 32000) # 16000, 24000, 32000 + bitrate = params.pop("bitrate", 128000) # For MP3: 64000, 128000, 192000, 256000 + channel = params.pop("channel", 1) # 1 for mono, 2 for stereo + + # Output format: 'url' or 'hex' (default is 'hex') + output_format = params.pop("output_format", "hex") + + request_body: Dict[str, Any] = { + "model": model, + "text": input, + "stream": False, # HTTP endpoint doesn't support streaming + "output_format": output_format, # 'url' or 'hex' + "voice_setting": { + "voice_id": voice_id, + "speed": speed, + "vol": vol, + "pitch": pitch, + }, + "audio_setting": { + "sample_rate": sample_rate, + "bitrate": bitrate, + "format": audio_format, + "channel": channel, + }, + } + + # Handle any remaining parameters from extra_body + extra_body = params.pop("extra_body", None) + if isinstance(extra_body, dict): + for key, value in extra_body.items(): + if value is not None and key not in request_body: + request_body[key] = value + + return TextToSpeechRequestData( + dict_body=request_body, + headers={"Content-Type": "application/json"}, + ) + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> "HttpxBinaryResponseContent": + """ + Transform MiniMax response to standard format. + + MiniMax returns JSON with base64-encoded audio data: + { + "base_resp": {"status_code": 0, "status_msg": "success"}, + "audio_file": "", + "extra_info": {...} + } + + We need to decode the base64 audio and return it as binary content. + """ + import base64 + import json + + from litellm.types.llms.openai import HttpxBinaryResponseContent + + try: + # Parse JSON response + response_json = raw_response.json() + + # MiniMax API response format check + # The API can return different structures: + # 1. {"data": {"audio": "..."}, "status": 0, ...} for HTTP endpoint + # 2. {"base_resp": {"status_code": 0, ...}, "audio_file": "..."} for older versions + + # Check for errors - MiniMax uses "status" field in HTTP endpoint response + # status: 0 = success, 2 = invalid api key, etc. + status = response_json.get("status") + if status is not None and status != 0: + ced = response_json.get("ced", "Unknown error") + error_detail = ced if ced else f"API returned status {status}" + raise MinimaxException( + status_code=raw_response.status_code, + message=f"MiniMax TTS error: {error_detail}", + headers=dict(raw_response.headers), + ) + + # Extract audio data + # MiniMax returns audio in "data" field + data = response_json.get("data", {}) + + # Check if response contains a URL (output_format='url') + audio_url = data.get("audio_url", None) + if audio_url: + # If URL format is used, we need to fetch the audio from the URL + # For now, return a response indicating URL mode (TODO: fetch audio from URL) + raise MinimaxException( + status_code=500, + message=f"URL output format is not yet supported. Use 'hex' format or fetch from URL: {audio_url}", + headers=dict(raw_response.headers), + ) + + # Get hex-encoded audio data + audio_hex = data.get("audio", "") or response_json.get("audio_file", "") + + if not audio_hex: + raise MinimaxException( + status_code=500, + message=f"No audio data in MiniMax response. Response keys: {list(response_json.keys())}", + headers=dict(raw_response.headers), + ) + + # MiniMax returns hex-encoded audio by default + # Try hex decoding first, fall back to base64 if that fails + try: + audio_bytes = bytes.fromhex(audio_hex) + except ValueError: + # If hex decoding fails, try base64 (for older API versions) + try: + audio_bytes = base64.b64decode(audio_hex) + except Exception as e: + raise MinimaxException( + status_code=500, + message=f"Failed to decode audio data: {str(e)}", + headers=dict(raw_response.headers), + ) + + # Create a new response with binary audio content + # We need to create a response that contains the decoded audio bytes + # Remove gzip encoding headers to avoid decompression issues + clean_headers = dict(raw_response.headers) + clean_headers.pop('content-encoding', None) + clean_headers.pop('transfer-encoding', None) + clean_headers['content-length'] = str(len(audio_bytes)) + + # Create a new response object with the binary content + binary_response = httpx.Response( + status_code=200, + headers=clean_headers, + content=audio_bytes, + request=raw_response.request, + ) + + return HttpxBinaryResponseContent(binary_response) + + except json.JSONDecodeError as e: + raise MinimaxException( + status_code=500, + message=f"Failed to parse MiniMax response: {str(e)}", + headers=dict(raw_response.headers), + ) + except Exception as e: + if isinstance(e, MinimaxException): + raise + raise MinimaxException( + status_code=500, + message=f"Error processing MiniMax response: {str(e)}", + headers=dict(raw_response.headers), + ) + + def get_complete_url( + self, + model: str, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Construct the MiniMax endpoint URL. + """ + base_url = ( + api_base + or get_secret_str("MINIMAX_API_BASE") + or self.TTS_BASE_URL + ) + base_url = base_url.rstrip("/") + + # MiniMax uses a simple endpoint path + url = f"{base_url}{self.TTS_ENDPOINT_PATH}" + + return url + diff --git a/litellm/main.py b/litellm/main.py index fe2c2f333fc..f4d8609b21e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -110,17 +110,30 @@ from litellm.types.utils import ( RawRequestTypedDict, StreamingChoices, ) +from litellm.types.utils import ( + ModelResponseStream, + RawRequestTypedDict, + StreamingChoices, +) from litellm.utils import ( + Choices, Choices, CustomStreamWrapper, EmbeddingResponse, Message, ModelResponse, + EmbeddingResponse, + Message, + ModelResponse, ProviderConfigManager, TextChoices, TextCompletionResponse, TextCompletionStreamWrapper, TranscriptionResponse, + TextChoices, + TextCompletionResponse, + TextCompletionStreamWrapper, + TranscriptionResponse, Usage, _get_model_info_helper, add_provider_specific_params_to_optional_params, @@ -6507,6 +6520,38 @@ def speech( # noqa: PLR0915 api_key=api_key, **kwargs, ) + elif custom_llm_provider == "minimax": + from litellm.llms.minimax.text_to_speech.transformation import ( + MinimaxTextToSpeechConfig, + ) + + # MiniMax Text-to-Speech + if text_to_speech_provider_config is None: + text_to_speech_provider_config = MinimaxTextToSpeechConfig() + + minimax_config = cast( + MinimaxTextToSpeechConfig, text_to_speech_provider_config + ) + + if api_base is not None: + litellm_params_dict["api_base"] = api_base + if api_key is not None: + litellm_params_dict["api_key"] = api_key + + response = base_llm_http_handler.text_to_speech_handler( + model=model, + input=input, + voice=voice, + text_to_speech_provider_config=minimax_config, + text_to_speech_optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + _is_async=aspeech or False, + ) elif custom_llm_provider == "aws_polly": from litellm.llms.aws_polly.text_to_speech.transformation import ( AWSPollyTextToSpeechConfig, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7f47ede07d1..513a4a554e0 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -19390,6 +19390,38 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax/speech-02-hd": { + "input_cost_per_character": 0.0001, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "minimax/speech-02-turbo": { + "input_cost_per_character": 0.00006, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "minimax/speech-2.6-hd": { + "input_cost_per_character": 0.0001, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "minimax/speech-2.6-turbo": { + "input_cost_per_character": 0.00006, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, "minimax/MiniMax-M2.1": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, diff --git a/litellm/utils.py b/litellm/utils.py index a71b44facfc..102df5d595e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8261,6 +8261,12 @@ class ProviderConfigManager: ) return VertexAITextToSpeechConfig() + elif litellm.LlmProviders.MINIMAX == provider: + from litellm.llms.minimax.text_to_speech.transformation import ( + MinimaxTextToSpeechConfig, + ) + + return MinimaxTextToSpeechConfig() elif litellm.LlmProviders.AWS_POLLY == provider: from litellm.llms.aws_polly.text_to_speech.transformation import ( AWSPollyTextToSpeechConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7f47ede07d1..513a4a554e0 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -19390,6 +19390,38 @@ "output_cost_per_token": 1.2e-06, "supports_system_messages": true }, + "minimax/speech-02-hd": { + "input_cost_per_character": 0.0001, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "minimax/speech-02-turbo": { + "input_cost_per_character": 0.00006, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "minimax/speech-2.6-hd": { + "input_cost_per_character": 0.0001, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "minimax/speech-2.6-turbo": { + "input_cost_per_character": 0.00006, + "litellm_provider": "minimax", + "mode": "audio_speech", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, "minimax/MiniMax-M2.1": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, diff --git a/tests/llm_translation/test_minimax_tts.py b/tests/llm_translation/test_minimax_tts.py new file mode 100644 index 00000000000..88ddf9be0b1 --- /dev/null +++ b/tests/llm_translation/test_minimax_tts.py @@ -0,0 +1,371 @@ +""" +Tests for MiniMax Text-to-Speech integration +""" + +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path + +import litellm +from litellm import speech +from litellm.llms.minimax.text_to_speech.transformation import ( + MinimaxTextToSpeechConfig, +) + + +class TestMinimaxTextToSpeechConfig: + """Test MiniMax TTS configuration and parameter mapping""" + + def test_get_supported_openai_params(self): + """Test that supported OpenAI params are correctly defined""" + config = MinimaxTextToSpeechConfig() + supported_params = config.get_supported_openai_params("speech-2.6-hd") + + assert "voice" in supported_params + assert "response_format" in supported_params + assert "speed" in supported_params + + def test_voice_mapping(self): + """Test OpenAI voice to MiniMax voice_id mapping""" + config = MinimaxTextToSpeechConfig() + + # Test OpenAI voice mappings + assert config._extract_voice_id("alloy") == "male-qn-qingse" + assert config._extract_voice_id("echo") == "male-qn-jingying" + assert config._extract_voice_id("nova") == "female-yujie" + + # Test custom voice passthrough + assert config._extract_voice_id("custom-voice-id") == "custom-voice-id" + + def test_format_mapping(self): + """Test response format mapping""" + config = MinimaxTextToSpeechConfig() + + assert config.FORMAT_MAPPINGS["mp3"] == "mp3" + assert config.FORMAT_MAPPINGS["pcm"] == "pcm" + assert config.FORMAT_MAPPINGS["wav"] == "wav" + assert config.FORMAT_MAPPINGS["flac"] == "flac" + + def test_map_openai_params_basic(self): + """Test basic parameter mapping from OpenAI to MiniMax format""" + config = MinimaxTextToSpeechConfig() + + optional_params = { + "response_format": "mp3", + "speed": 1.5, + } + + voice, mapped_params = config.map_openai_params( + model="speech-2.6-hd", + optional_params=optional_params, + voice="alloy", + ) + + assert voice == "male-qn-qingse" + assert mapped_params["format"] == "mp3" + assert mapped_params["speed"] == 1.5 + assert mapped_params["voice_id"] == "male-qn-qingse" + + def test_map_openai_params_speed_clamping(self): + """Test that speed is clamped to MiniMax's supported range""" + config = MinimaxTextToSpeechConfig() + + # Test speed too high + optional_params = {"speed": 5.0} + _, mapped_params = config.map_openai_params( + model="speech-2.6-hd", + optional_params=optional_params, + voice="alloy", + ) + assert mapped_params["speed"] == 2.0 # Clamped to max + + # Test speed too low + optional_params = {"speed": 0.1} + _, mapped_params = config.map_openai_params( + model="speech-2.6-hd", + optional_params=optional_params, + voice="alloy", + ) + assert mapped_params["speed"] == 0.5 # Clamped to min + + def test_map_openai_params_with_extra_body(self): + """Test that extra_body parameters are passed through""" + config = MinimaxTextToSpeechConfig() + + optional_params = { + "extra_body": { + "vol": 1.5, + "pitch": 2, + "sample_rate": 24000, + } + } + + _, mapped_params = config.map_openai_params( + model="speech-2.6-hd", + optional_params=optional_params, + voice="alloy", + ) + + assert mapped_params["vol"] == 1.5 + assert mapped_params["pitch"] == 2 + assert mapped_params["sample_rate"] == 24000 + + def test_validate_environment_with_api_key(self): + """Test environment validation with API key""" + config = MinimaxTextToSpeechConfig() + headers = {} + + result_headers = config.validate_environment( + headers=headers, + model="speech-2.6-hd", + api_key="test-api-key", + ) + + assert "Authorization" in result_headers + assert result_headers["Authorization"] == "Bearer test-api-key" + assert result_headers["Content-Type"] == "application/json" + + def test_validate_environment_missing_api_key(self): + """Test that validation fails without API key""" + config = MinimaxTextToSpeechConfig() + headers = {} + + # Mock both litellm.api_key and get_secret_str to return None + import litellm + from unittest.mock import patch + + original_api_key = litellm.api_key + try: + litellm.api_key = None + with patch("litellm.llms.minimax.text_to_speech.transformation.get_secret_str", return_value=None): + with pytest.raises(ValueError, match="MiniMax API key is required"): + config.validate_environment( + headers=headers, + model="speech-2.6-hd", + api_key=None, + ) + finally: + litellm.api_key = original_api_key + + def test_transform_text_to_speech_request(self): + """Test request transformation to MiniMax format""" + config = MinimaxTextToSpeechConfig() + + optional_params = { + "voice_id": "male-qn-qingse", + "speed": 1.2, + "format": "mp3", + "vol": 1.0, + "pitch": 0, + "sample_rate": 32000, + "bitrate": 128000, + "channel": 1, + } + + result = config.transform_text_to_speech_request( + model="speech-2.6-hd", + input="Hello, world!", + voice="male-qn-qingse", + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "dict_body" in result + body = result["dict_body"] + + assert body["model"] == "speech-2.6-hd" + assert body["text"] == "Hello, world!" + assert body["stream"] is False + assert body["voice_setting"]["voice_id"] == "male-qn-qingse" + assert body["voice_setting"]["speed"] == 1.2 + assert body["audio_setting"]["format"] == "mp3" + assert body["audio_setting"]["sample_rate"] == 32000 + + def test_get_complete_url(self): + """Test URL construction""" + config = MinimaxTextToSpeechConfig() + + url = config.get_complete_url( + model="speech-2.6-hd", + api_base=None, + litellm_params={}, + ) + + assert url == "https://api.minimax.io/v1/t2a_v2" + + def test_get_complete_url_custom_base(self): + """Test URL construction with custom API base""" + config = MinimaxTextToSpeechConfig() + + url = config.get_complete_url( + model="speech-2.6-hd", + api_base="https://custom.api.com", + litellm_params={}, + ) + + assert url == "https://custom.api.com/v1/t2a_v2" + + +class TestMinimaxSpeechIntegration: + """Integration tests for MiniMax TTS via litellm.speech()""" + + @pytest.mark.skip(reason="Requires MiniMax API key") + def test_speech_basic(self): + """Test basic speech synthesis call""" + # This test requires a real API key + os.environ["MINIMAX_API_KEY"] = "your-api-key-here" + + speech_file_path = Path(__file__).parent / "test_minimax_speech.mp3" + + response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="Hello, this is a test of MiniMax text to speech.", + ) + + response.stream_to_file(speech_file_path) + + # Verify file was created + assert speech_file_path.exists() + assert speech_file_path.stat().st_size > 0 + + # Clean up + speech_file_path.unlink() + + @pytest.mark.skip(reason="Requires MiniMax API key") + def test_speech_with_custom_params(self): + """Test speech synthesis with custom parameters""" + os.environ["MINIMAX_API_KEY"] = "your-api-key-here" + + speech_file_path = Path(__file__).parent / "test_minimax_speech_custom.mp3" + + response = speech( + model="minimax/speech-2.6-turbo", + voice="nova", + input="Testing custom parameters.", + speed=1.5, + response_format="mp3", + extra_body={ + "vol": 1.2, + "pitch": 1, + "sample_rate": 24000, + }, + ) + + response.stream_to_file(speech_file_path) + + # Verify file was created + assert speech_file_path.exists() + assert speech_file_path.stat().st_size > 0 + + # Clean up + speech_file_path.unlink() + + def test_speech_mock_response(self): + """Test speech synthesis with mocked response""" + from unittest.mock import MagicMock, patch + + # Create mock audio data (hex-encoded as MiniMax returns) + mock_audio_bytes = b"fake audio data for testing" + mock_audio_hex = mock_audio_bytes.hex() + + mock_response_json = { + "data": { + "audio": mock_audio_hex, + "status": 0, + "ced": "" + }, + "extra_info": {}, + } + + with patch("litellm.llms.custom_httpx.llm_http_handler.BaseLLMHTTPHandler.text_to_speech_handler") as mock_tts: + # Create a mock httpx.Response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = mock_response_json + mock_response.content = mock_audio_bytes + + # Mock the response wrapper + from litellm.types.llms.openai import HttpxBinaryResponseContent + mock_binary_response = HttpxBinaryResponseContent(mock_response) + mock_tts.return_value = mock_binary_response + + # This would normally make a real API call + # but we're mocking it for testing + response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="Test input", + api_key="test-key", + ) + + # Verify the mock was called + assert mock_tts.called + + +class TestMinimaxProviderRegistration: + """Test that MiniMax is properly registered as a provider""" + + def test_minimax_in_llm_providers(self): + """Test that MINIMAX is in LlmProviders enum""" + from litellm.types.utils import LlmProviders + + assert hasattr(LlmProviders, "MINIMAX") + assert LlmProviders.MINIMAX.value == "minimax" + + def test_minimax_in_provider_list(self): + """Test that minimax is in the provider list""" + assert litellm.LlmProviders.MINIMAX in litellm.provider_list + + def test_get_provider_text_to_speech_config(self): + """Test that MiniMax TTS config can be retrieved""" + from litellm.utils import ProviderConfigManager + + config = ProviderConfigManager.get_provider_text_to_speech_config( + model="speech-2.6-hd", + provider=litellm.LlmProviders.MINIMAX, + ) + + assert config is not None + assert isinstance(config, MinimaxTextToSpeechConfig) + + def test_get_llm_provider_minimax(self): + """Test that get_llm_provider correctly identifies MiniMax models""" + from litellm import get_llm_provider + + model, provider, api_key, api_base = get_llm_provider( + model="minimax/speech-2.6-hd" + ) + + assert model == "speech-2.6-hd" + assert provider == "minimax" + + +if __name__ == "__main__": + # Run basic tests + test_config = TestMinimaxTextToSpeechConfig() + test_config.test_get_supported_openai_params() + test_config.test_voice_mapping() + test_config.test_format_mapping() + test_config.test_map_openai_params_basic() + test_config.test_map_openai_params_speed_clamping() + test_config.test_transform_text_to_speech_request() + test_config.test_get_complete_url() + + test_registration = TestMinimaxProviderRegistration() + test_registration.test_minimax_in_llm_providers() + test_registration.test_minimax_in_provider_list() + test_registration.test_get_provider_text_to_speech_config() + test_registration.test_get_llm_provider_minimax() + + print("All basic tests passed!") + From 980860a8277dc56229a79c941fe8b744a30da98b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 23 Dec 2025 21:30:23 +0530 Subject: [PATCH 038/388] Fix : mypy error --- litellm/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index f4d8609b21e..6e069988726 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6538,10 +6538,18 @@ def speech( # noqa: PLR0915 if api_key is not None: litellm_params_dict["api_key"] = api_key + # Convert voice to string if it's a dict (minimax handler expects Optional[str]) + voice_str: Optional[str] = None + if isinstance(voice, str): + voice_str = voice + elif isinstance(voice, dict): + # Extract voice_id from dict if needed + voice_str = voice.get("voice_id") or voice.get("id") or voice.get("name") + response = base_llm_http_handler.text_to_speech_handler( model=model, input=input, - voice=voice, + voice=voice_str, text_to_speech_provider_config=minimax_config, text_to_speech_optional_params=optional_params, custom_llm_provider=custom_llm_provider, From f284fc716defba696ba88dde40da51d89a80a59a Mon Sep 17 00:00:00 2001 From: vasilisazayka Date: Tue, 23 Dec 2025 20:59:51 +0400 Subject: [PATCH 039/388] fix(sap): add sap as provider for list in add credentials component in proxy ui, add sap logo (#18375) --- .../public/assets/logos/sap.png | Bin 0 -> 200176 bytes .../src/components/provider_info_helpers.tsx | 3 +++ 2 files changed, 3 insertions(+) create mode 100644 ui/litellm-dashboard/public/assets/logos/sap.png diff --git a/ui/litellm-dashboard/public/assets/logos/sap.png b/ui/litellm-dashboard/public/assets/logos/sap.png new file mode 100644 index 0000000000000000000000000000000000000000..7d3c4604c4cd1f84c57fa9bbaca4104f5ead66dc GIT binary patch literal 200176 zcmeEubyQSc)HjTS5-KRAh%kyEjS5I9jDZ0Xf`qgpA|l<*7+_E;A(93ljdU{xp~!%s zbV)cgBHiDaQQ@NCGtcw>@x5!k<65qD*5%H*`|SMfefBwb6y;BDqGX~ZAtBjx`qXhH z5|WKwBqS6Qi13di z`Cj1hBr56_#~oJ$xr`Ez3^Luy^@zdQ%Z?}RTxZ+&T#5eB=!0= zUW|r0-0Pwt%|&u|nh>^E==LIi6snv0=!fS)s0*mg>((uvyZPwxu5~xuCD*+v6nhsX zZJ?;XE!b=OHu4eDD~U4BEC;u$0?vsVsGq(pCr82!#^fZVo<<~aFd_y2nZSP%5?Cbc z*OQH1k?VeqDJF1#MM8okIeq-dIeXH9rk9&Y5F6q9^%lcO_mkF1jH{_OhRn8_ z!^vq7NOuxa*uNY&QzRq@Z28we`}r{iq+jQj|M+D%=@EJK0rvkwGz|h#HnQ$tuJjjw zuDc`l>IHB8Pu!xR>6-elygGvPqfI$O@r#Uq$2D@xxqmNP{HIWWR}|{e|B+d6G1vc~ zRJac#0k7b^%Kug$f3oK~v($f63?zb_+!7YZ^&h2l#E132DF%WD4i_UW_y2EFaz~U> z|2M^;A%|TLbvKatZ&Cv0yW!s{;u4ia^ZgeJN;F?WwTx)K|H5+-&6i*sBu3vQ&;8E> zV)P{xwh*K5zW^RY@chrF_XDRB!84%-2+%qaJQHk##N78^WOu~em(ZdCG57r!X+N=a zMW_uDOIL&%ATA9emaYiRoQb6?g41R=sHPGt&xAQFV(E%d0|aRhv2;bS4H8RNgr>~^ zg%L|vgl5jf(iOpJGdXeJm(cwi;=b>{2!h1Y6~RFePApv!hC=a9jaa%OG;=1Fu85_p zHJuPK_a!v!6La5x5%!6>@0u3(d!|RseShc3UjiF3_a)}Ot4DJpcqW49HA6XZ1+d6>7f1x zSLyuM8>xumC0r-LF*l-k39@IR^%5FTf$sskN<`}=$Xyei^Xe#N)$|(NctVh^{;8=% z>;01_Yd#@bFQEaHXuU-1T~k+x(2Af4=uSd(&TG2jKh2BioYyqC-!g8ZbN*9%)_g*Q zR=;V_nw%n*HU3rEomjOZOtFYnD?tb zL=?TmihmvuJ2!}(8^q2HV$bux06WCvWQ2kp1d@oN|Ak@^Q8b~7MMTm60uTNlM$v@o z(@$;^k@G(T<|URAkuwoF6Ol6!IsXfQLqyI?-t!*^MCAN06zl&Na=wTivgqP|O5!!K z%`c_P5B-Ezm6WvAzX5*l;*JE8Jx90FkUzP9>sqC+Q{hl}-FTe*`Zt3^ampg|G+p)o zVpDDfC-C=Wd2Rg{01y(?iQgqKXp_Q-d|sh;tA_d|B_!$f4iN2#Xh&;ufao{Yuo_}KBKnOrt!63gE)9rwM6{zd zI6$-`q8+Wt0ixen!)l1}m*_Xvw3?-`yEGu$5z&s;-~iE%h<3Cl2Z(-S4XYu>U!vbw z(`uH&?$Us0M?^bXg9Ai6BHGcK93c9QHLQjhe~Er$O{-Z7yGsM29TDwl4Gs|Ph-gP^ za^QcZ-`L%DxBb^$0Dr%&g{a$WN_HtUEDiow%9Qc_!s2U+d_H-vzvPGF(iWJGfwql+cp<1jEHPrdrnf=8%pQJub;oP zGzGm0uY3hL@+ivOHJZ8RB9^@1uffLM-y};MTw=HP}2?8ZZ^>`PDnX%Iw1)2Vgzqa9>cd8A)jScdNI zfBNYX6DQ$sMJG?qp_bNy{sy}q`X!=_lwx_>KgowY(NiVek4X^d*IAk~jf&jzvfG8p z<+c5t9Mmqdx1z@+?;YE+G_Cyuz>L#pJpRhVrMJNV-3=BJlr|=}LjGL!)zqg=A!9__D*&+I{z1^4XNzO}@ie;AZ7&~$puOUj>I(y8`Xz|I(%U_yC8 z3Ns|XZGl0Lo?TYx8x-W0>z!yycM-afX~2uog4i8P{7m3Ol%1cQZwdbg`j^-peje#H z`T1o)!OE!vlr!I~|MD``retuji6!~`<3LLh>1B1~jIQ+3n|g}C6Gc*Y zsxQyEDg&0Jl_CX*SZMulXPN5I-H0+V=?cTY|8Qvrpx-;e96Mhq0$#F}rps%b zMgG_lcm82;15B~r4N2weJxv$y%(o=D2r~doT zmcQtZJ_$JCyKjySLMt^Txh2b`k(0mNBIHl25(XCTwQ}JoeneUCWl4h7jF$4Y1&8xF zwPjTB?F@)YhS$e4R{i2CtVm%6O5xi2{`9gMc_9#G+rQ9F^M@h20M)Onv zH7<#>5)2ax06QmUEHTTvthW&mv2D1WjzA(1DffXZX1gM_>}>S@tvCM`3zSotdhx^( zE8NjWK&my(&UgPo5KHQ73nbFea6^oLNe59V5FOV^`8*@o>lom0m&F(#Wa;^gplkpq zRKFi-AiV^~M{}8%iHIi$QU3MNNkY{^j|!*-!+Y0dM2>a@tQc=?_9k$Ol$$^xcMNS$ zFH^0yg{DjER*Ns8x2d45fL??v*V#ef4RD+5AdLB|a_I|4)ep6k%9qT~&JpYGlw8&?kfZDc*eU$&rQ6HzFL8X;!3oAGsgmiA zTiXoBvMeQ+sd5719hfHziy(BO%@>(vA%vp?fo%9P<{4J~>XLFU545xZ1hY`mJ+-`M zG%g$5f>>Ni510S6I#neg8hFhflwUR%y~|*V56S(e1PbU8$k+r&I&+zp$VR;a#&Sc- zuajV7aRGYjLhA+r@_G`03fo%OJFAnu%!FUM!;b)7l;mmhvI9f5^MX0fJ4>GVCno%w zO%rHp=BEiw$jT3K0GM#5q>Vs(LF#dWr6^W@*|$7LAOnc;^K<;m&iSwOpj<$rGch7B ze|qUPJ$kSZZh^t@1pviOEL_Jd z6Rjr?hzK3-Q(d0)??e;;A|@4jmJI|R4baY+*>rwFiI~Sllv!JO`6Uwd$N&+P$pb!w z5`haSCw;qLEWiIRAs1>3KwNBvcLgEJY_0|%sM^~ih)09i@t+!tRsq34?ZV`Q8)tYWa{%S3>C6xEH{cq0`K3 zy1yykf?1N(0rBB=i#A1=9K+nb)Ewg<5L>sy*E?J$SGxI#>!ZfI1MKC2ml{+#gI>qk zW~k=8Z~dH9o0pNn&90F;5Wttm`L0$&gXggfg6sP3BiBjbyJ-+xIIzJ!PDAU~Ur@5E z%r}2Ctg2l&G&v-Jopr@-iisIn82)A1KRATyBPZ`XXP9WSoasfLfx*NKG~^0azPaRw zejh|4VR;8P75hr2a9j=BNfxIO3*iIJZJM$XZV1DdbZ)f)Dv;{Zg-})}#ceU&-#8!; zDRLK}Y10KE^4~wc+AO$tcvXAp{1;#0-4&v8cz!r_*WpIkiyQMkV1SFru8Rl0{0-i()MsLE=vWWs0ik`&L5geU^wwp zBfSuspX8Mbnf;1C5bd))-ViqbouXY{ozf8XL4tynWn9X8M+krZDibRBO$!b8Ym^W@ zu-t|4dw9=4ak&EZcj4USLwG3Y$QLTeQn;|wk7S$u0Aj^aN%>=t5lu=@gG-dCmVO+k@hY&h{Usoa>O^H!dW)6fMoN<61XVv6e z{_->EpRS|;cHB6!X}Rq0jx@rVY+gDO#KSKe^q-`mt3gBvH0_pwaD$w@%YM`E?dR}fhLLmi|f?@Zs)_6f2Oe5={986zo}~3Ex=Cxz$05A@B1qPDsJA3V<(Ub^3Ij! z5a|GTS&=4ufKYF-8+Z!_+lLUaqqqw=PfET_RS19o5aaT?IE%n4rt8!_A*)E&1*@1h zj)W1o3ZdieNpd$v^r@5XXc(S6RwB3(LTjlb5!l!KLxdX36L{)AjIUI->b4lg_-J(q z5qEK6@>77a2!ZU;VWcS;QG%`>X$+j0M!yNc`hE`R`;*Z1P@ji>0IWaoZP#uHWq(t( zE(H)4+>D<>-;V<@OxfF}mYuMbu+!3%a5Y@N3XS(iLAj{-@Cx{75W>=3x=g{&20IK_i!{$GU zUiFCvFHri6<~2}+(Y_TJP7r#8X4PN(DV^^KfCW8O6xTuDZv|k`q2{$3gh$JU0@b1) zd@c(ErZW_H{0Q`Yaul<`3NHU;hAyC7dt<1c@j|%4EdUf_I*5b>^6XFYm!5p*1nVeO zIZF#&hYrB!BQk#yyy_Dq28h)%6Y(BW!gEK90d`{DrwGlM262cZrZ5NUb5ewWBZ?Bs zA+&Zc2aMH7{3XTG+Q6WS2F&5HkLl*$G#j#XHQ?2)3|zyPH!?t#Se25zD_yBc3o52n zS@Gi|+3p}2CR1!)-W~^YFoW38WpCk6Sp5q*4b=R7QHeMN7)Lq-mUe3@?}V!D?;(1b zEJYd+Ca?4QAAqvd9A_zFh(bcGjR%NUfyIg?ogi=yv@}=>zf32D$#8Ny z0E}a3%AcPiOiAB^D5V@2@M!~N5)}rTve@r^{e%_3P=UCJ@85I@D#I&}0cxnZvWdVV zF-ioiq&58;J%kGQU4CG0WAx{wpbw!M2t^2_upFSH@8-91`9bxud@D$Y8G`OXCAIQr zP^u9rhk#_e9daH+=a8nMDdT$09eW@8aK+D{Gop?HcFR4@nsy^4o_`8M=V z<`)1zsPeLdpi4IZJmspn+DBNQ5-Jm?9szA#P>x**2I^7%LXI$;j||3{Y<{leib&rC zSlT-DMS?IrC+F1!^{a+U;VWeYTnC-wG~IGN{x2MLN9F-K8Vg`wp-~yxPnN486}A00 z4MU|;TH7AqXl-|5vENW3kuA5zVPaySPP_LtW^Dhiu=}PpR^rtS!3!}HGbcSiRGj49 z^L%>x*34k@>!X(@D?aeOy_PGGml)C8Un4kbarN>p`C2oqzWCv

om;3Njw&OxR6W zpu7MXFXH!%7s%F+3gz#6=VPIU7xnYBBA${NqKr-$vg4 zyj@OhR-`jbc2}5kvZqyQB&H#`a=oojSg`N27;HzGn4xXkNU#5l#i~J6?+(yD*_)Ot z>OV5LwCc6==dCLZI@=!^7<^u=ivHm-?fv`f>EqYu>F;(q99)PQVpaGOa-S|e)Bmnb#th|(A2s@Km7lilvg<)fH6e!};fj9+Z5z zKnE2ZU})rN*p0(sFOBl|9`uQ};+4qLKQ`m2(l`=*tS9<(-&txQgICc5%?|g^%ga}m ziulNvmX}`c&l1=fb~;P2cQ!b^6Vogm-fjBZta#o24vH8LC+U?~p+eALL-^G&8K$k2 zC5nfk%r>tf_o>Li zJ9)Xo*fYA**fXZH%PY(V!)Qef{mu{Hf^r5`wl4jh=b-XJ9ReA1*9H1dPy+?O#cWXz zv(0XhY+!8K>)3C|FtDv>=n$5>jX@aaVEn=9XrdRFFd z`6MQsWOZ!q_$Y6n#$!^f2iqg^W4#V z^Go94VmqoE+&IF;>^bBs>SeGA_cdV-jP`m>idw?84<`?NRwF?vNZN|}C{!{=DzSgn z)|)?N*IZ;W_0)i$S8E3AB`f9l@j{!u-u*B7A4eE-^zM7sOx@G6FrY4_pPcmgbgM|+ zc}|)s`&)`xbKcaHnU>=#0S2c9vWT%jiAq(s37bc*%t@>bCN6{3CKEOjR+fR95pr3h zy^BoP>bDJbR=xef))FBsAFdCc?{N`t32qS)%%d@^p6r$V34RoD(T-A}uomjeQ*HxX zw;Rc_+((30&~o~+rw4&Xb_Q-*XHhqw7JkAV9YB8RNZxRs)d3GjW3>Zb*sU+9 z1|%eR6wP*>Wr8QE>&;)Wo3j&1&hD_-h>UyzJ8zC1``{dkfr+P$_MT~rX&RV)$Teku z;Ba6|_r&Q9<=dq{-9$Rb)ni;HE<+`; zIw(RlQzWqV`q)K_$Qxom#}8b8m`A2m#ZI<9SE4gI{f4sb<&B5nd@L8E;yzCZz$q7q1&9`Gk*LLVZ5J5}a! z>Kv*#_R@G%&vBiKf)&SpZqz~Yd>h8Ff+pQ2X>5IP&Qx`IOvlTh>R{6d3DKQYCNC=d z;c>fDEF&32qO@yqwENAw&4$O`pUR4ms(-QT$6gj6U779I;B5nO5sV=RscH?MgsCa1 zV*~UBdk=32NQOdY)}FYC223KB#fr=OA(>U7g*AtcjG=8vm;iIQkBgDr z&YyPr%@){cmeZ8aFNcl~Z2Wu(KPw~Fl^s+_X|jV}wxu_kJUUU|E# zeoBPjEtj+wZGrSR{k_4{_9JE1l-PWRDu-Pz^K(yn9nDLl#mUsQ*m0N}`38i1Ci$yK zXcYF>0omD@Y{C)fLmKrsw7nbOP}}2M<{lTXcFZX-==6yuu_@A{9Ff>Kx~kKLw#2Hae0)_rZaG@b>h;6T##bmP53qG1&M`(M~vGI5R*{iVae;f(C}w z(>Q6Go<6Va$&@FVzwQFi@6$G7U z>vmBzm>bG(_CG!f2TIQ*km4Y1b_wfgw#|1yIkRZ2I#vWGF2E9OG~*n*_kZ0s-(!W9F-aCTZ@-CuEqyFVkO)Pjo(3nZGq8 zJp#YW;pg&x!o)}ymt}$P#4t}bztVZ! z+j`PUe|Tn`q0Ar*pz#i0CFdM zd!~>w@U_9YB`bWbFi0NbUSe{g2`Dm(Om&k@u~RZj;_z*Kkx-c;I-v+Ql*+M|k?=9> z0SSf54|R5!X<#PK6SrtvQ-Yq^D9ISZC$6~-Zc0cDuHaRn&$ehmdX3qXe5@H&VXwm1 zkJFKnc>s2)jeoAB)?RG92^>Ju1}he)>2rqyUr%k2$7DoqRmKM_lq?l_r&ePrv(GX52xM+84krwsKOykeTKseS4@S#TQ!}-JI{;ml_;{X zj%FgyO-Mki>++fENGUcf2)Bx(1~ygaWqH3fbD1{p=Mer;+B*f?QR>)-&9n2VJ*;!+ zz)sz%65}tS0)^H}h_-lqc}xShb7*Nv3l(!@Iw(3tX=`{x5n|+HSZ_hJMO*EcDMiYT zYuXK(a&T>0&J<)~8e5XWh1`L;Ji9Z;CyW};@5A1=mJGfwPDM5(*9+(0lVVqzQq^-y zuepv6)uQ|cCMI`Zm=ov2c~9!SxuS3h`=-vTi_RJ~vbvPo zGm8uFa2sj9GgV{gTO6dzfQjuhetOMiIGBk`Q+DRvOC*D8t57R#CF^61MSu+6fVe5F zpSN6#;h5>emYQm9*&BoHOfIrGu)uxGN^mgZ$xh#&Nz?}&4^1Uwha%*EuOD@ZMQ)IQxYL%cj%Ut zPibdzs3caVZ7Zh-8uPBT?5N>pSR!5hL)&-LZw;x_EoH3mLPD`qZ5*>MNvb$DF4%M! z8#`~*D6;p#PHlY4C>9VHYSytL7pKX#1aY57LNxM|m?z$(h~lE4xe|lSq^+Gz+8G})tMH*DA_{m;$(IVEF=71|hq@^ST1@RiyYJ-W zZh_gwh=@^j2_3E8hMTqO8tU3)XxKxS@lgpg?d{3TA+es{)ASY|&SHd|96NDBMBV|F z0-5wj8qkU|C^N$?e=jqW0Lw~Xeo>s&n*EgJZEuo|*33l^TNNo#cj&5SfenpYwT%SN zB#8lInH4hGfXb)5biUdtb7ObLA)5eoy>WB9=2TY%%lyo{t#+}(cu)NdIbl~1{A_Z> zdCo-vrG41SYNOH*2a+p|eci~&_vytyF8$!>$*a%Ce7-$rtds86H3~$_> zVt7>@5ySVL1FujgC@wvjo75Fil}b}waC7=n1+}xqI-q)_Q8%F4jV^cfUTV{%M5eJwTR2osXvrFGTUUTb5@Ggt^3^Emp113 z;czR<%dU6Wvb~U_z!|cdwzi+(NV$yGW`~-i0tEUN>gT&ky7ioy z%SvWR_WVvA8d~l)!i{sqsAycbh<7b|1&_Wl0FBGin|iPUBzgkk zev6?;b{EzuYh&?nOytuaJ@Edg^=jZs;A(EplG4|M0#m$S8Om z90Eh(iq(IVK!r{LJaoj*1FDWBs1kBeI~xdQ@-ZtiFgUYu#?V;*ocB?8^m$bH7{}Ob z=H|M}V|xudeTy6uSTGI?RbvgZ&0DYJqFZ;qIa|K)+KT4NGxSjG7F6iLnGk#__c2dZ z!^bUIrrQ!CN-IAUM?8<1b(?(m>ae)s2wY9Q^-@hcK&0?tOorlcjO3J$%fpHHzHnjmB}okv_*@bK2AMLx5B*J&KjXT z_qoDugOd`?>m!jZu)3t7c+ZH*h({)NTtDlGa`a`kE%RTEOZ%{DI%4#T-FKdi7*_7| z4Hj!MI7r{x4#axp2EaHws^nHw4DN!8!MQoINEfI}={lk}uBhfbxs%pn^6QgB{?FEf zw2w2z#K7!Yp78-m*V*?|p9D)g?IJ|Cx}`0iA!AQ(zJ>~WSn$#!A*P{bTLk(ktbt#y zA?fDg0+lC2LG6!oSdqN2x`ZL$fjOt)&{Gn{JH6W+=1vXJ%JY?f@4G>Q0+e+AK}q^) zAX>R^(KsVp@_MM-tr{nt`{g=Ep7fVIjftnF0Do%auL@cy>Upx0ZG5zMKBjXr{1B7d z_s{wkj+fuG~N`X(kH)K+D`Cr^6jE`F74BL9+i%(XY2D6?i@|`Nj zD!eswn>Kgu{#;YeQi|_HL9hbromStV0=3@AyTE!?v}_JT0~=xk%y!-rgyaXjZw%< zP327EnuoIkH!2ZpVWHP5|3I&`3MOL(Uaf=AC!D zT%8z}wAiyS*3h~CFjm{KSpC)9xg4JP7LiYy6}+D#F_H{;UlI8o)}*{` zv6MxN7FRob6?$xB1DC&4r{Ti;B7PF!XU%|1c24%~TMk(?2t_i~Sg_a6Z1I<9tDI3| zPtt%;OFOWV&H?-6M{_Dbw0Q_m()r9N<=oac^|>fROwzUK{i9R6j%`3iQ4Tan=6{ma zJ{f5Mn>5*ZcTE5B&+|0)+IoYh?B>ud@g1~nxm=QtZ`8+ZrbDXl;r{LbHZ$LK#=Te( zBtFxA^Fj3`=J-Ha2(W&9iDSoxVQqSey|WsUSrp)}RU1)X(#zkSY}>Cx&9u-xy0;^% z$XT@Tfo-J)P4*)^($f$zf9ctx5NQHySd3`NG1`~9C}8zdcg!fgChdw#%_447x9wwz zn5%v=6;|r|w)cGAr9!SQ9w4FBY7``>EMTemEpO~rIOr-QN@Iqm!7Rt(kfd$r-5yLK zu5b`}36_yQUw2s3qYqm%t59m@(rfPA2jn^I;>fPq%VTCZ?DM5eM_@BxMZKUfuiwMc zTR(P~yN>GZA&ZxCZ8}cb>04?hY0J4Vs^;@{`>Rl`1n=GCG%s*@l5m}N9h8c)c%-2Y zl``WBHKWti8?{%ME2qBZHJoh_wNL(KkkU8+l4tdbtes`I`z%o^N%zn)(PPlFWMhzRqG!;Ve;lpW(uX9Q>4vPh9zBem( z8U^G#0G;{HF-b2qcvSK8n1$&81D5)F&zCJV5C57H-1ULFZ^(6**DQ~Pm-pv!`cz8 z?vfYOLS;oAZj+#{l5d>&q=d9M1fa;*sZzV-yEbespRZ^IVSj+PQN`x_5&)&c#Bwz0 z$Z!Y(nM79WJ@(f3a5jev;+;qKWfk2dV-;LCEZ7D0`ePULJx`TNYpwOl9q&XsT&;DU z_Tq+Imx*IMEfv!4$OW=oyXzHnc=jcl}kyEjJU zcr~Jep?-gZ8uez^&Ym-AF$W~^3c(qGp14yu6>Qc@6XEz(eJ;L|qfikG{p>GPI0x4KJHV~f(UT42S+Wa1Ly9j%FP#z)fx zfbbUkZ;X5@6r@+e8Zx=2uE%Pbu)BuIegkwz_g=IRENC z$?5tpQ`B5Hr4DCR^=7o%iAeJ8#o1$O#U?w}Tb_UR6fj1S4lw3vg1qm7-gLB@?qxUYGApL~Fx%2t8{1H; zWHX8`P~!F14*OnkG!CE9?WP!z$*Y`i_DsI(nY^7(so^TX0Z2`N0UO_&S>na`%ZSvY zL52OmE3M`8NI;6*RA)BZ7FQn1mt>L2>lDQWphycCt`NBR{q6mdO;?_4ksr#IOWDOp z_2jkzMgC~qt5eg)KMRCa2-Lz5W@;giqf3W8#Zz??vXpC1+pXu^;Q-+az4z5V9x5Ft5ep z@p0pN2J(U6)EyV89?9X=h@P&ex6f<1P=EJ=LnEKzXGJwKvUHFjqo{BdlltNUXdFQJ zxq)qV|M!EH^LWG<}Fd!J!Kdd9gJ zrpbCdv=1h$jN!l;08{*1GhYw2j z=6}2Q0+sEQrX8NY5&v~04?MNVYOxYT)19;hiH|V2`!CA8K`=HAJi2Rz5X{Md?^JWg zt~f!(`3zl4e|V&*!0gyH=6)`HCr1!2k*09hL8}cNZ^xWX`_7)yhl725Sfck6A(;>| z+cOiznE|can!g(C!ygSfcN~|E&k6M2I4f!2qAo37%RvfLVax_LGc~fI?gZZ;rgtf% z{TN7t79g(geX%QAfHpB?J7~Mj-SZTAdl!|o-uId)jtlnjXe&?5+r`OGAG2D!`yY@G z$mMm~A3J-RL3zhZr}w4`I_vR^j8uVX_saFQjbiRhy*q6Dw5WW?C(UkCP$$t4@VjN< zdNFICe1CTZ#R`Yh`Pb*pE5CFvj{!$dC^^2}^BwS= zzIB~Gv6-D>p!Hzt^M0-4;DEs7we8$TgRlOmT1IWnHm*^e8<-U-@%T8TG#-UJ0DTF# z1)U8mM4@<)bU2l}>24pSt&a}dlV~IN- zn|9+d2h*@oc{2w)z?%U^b#I$QHVf%r(f?Wf3!@OJdmU;3-GKNImPrfx3a7MB2hF}Wo(yO9Vfn4lHa$el{& zGJx}qUf>fZBhDAEEO(_CzV8)-)l4)0yvO~yM@OH&<6bs4gRobF% zl^8K`nZfd6@(V{ZIr?xF4pbinJ|xruKo#$i8&{VPQWnnVz!mHYlKTk#1R~`O@W%$% zb3(!Tmi{2kSoT_!)x~q4{KvYp^B^G0wg;~>?4S-8S50`Tl6ML95(g1X_1@Lx`5`~z zb1$Ci{^lPAUh;G`9A-G|pXQJsQ*i4MPJZ`5BL*+5>%#IUz_&QPK{tR=pc32Ccfmki ztKFc7sa z2klC^7|FAM^{gu$H=g0|YzW6ygudcRxo!0uq2EFHvEl+z!YvuRtbVS(>Kuw%ON5Cv z&q!JsyAK2qIC-eRH^WL6U$8qjGq<5xnF+NaYg14(qc<*(YYEvPFyS}*!)c%xr~n%8 zh+)-W%rZ%l)xXy(chVib8|eH!5{E}n_QGXA5LV7*|Gc788N*nASeUuZs`TwNbvE7` zJm%haSb{ZTYG5H6bK%?#fP9%$x{8Ye&*6;zDT=7HrKgV!WM0TPv+>Nwt{L97f2}H z8Po|u6bnwihZHnNuY>A=zj}M_7}wW^9fDWGK>btaa^e=i9n=j_c$0D-QIwBYR%L@% zdp;;FXq~Gkm(i!goisT1rS$!E-ave3Y~%r0!>}{R7vnn@5|4@noVWJgS{4kH9m8;3 zY=z9LaJ%AxmLV$|rxI!F!7hQ+c#_S>hXtDTcqQgzoC+usr-?CS)R0W3(A=BCbBh*q zO(cd?oq;#S$|{$)r|qRh``wZe+!4uDjwSQ%iNjZNng_afY{W$p4842PLjPFV~;n@bb=E4@@j_ zBDeB&+s{M*;RbM!|0r!`rbj6z-@&Z6S@otut zjgK@MG#@>tmvKkF1p5wmGj!rsRs+9+_HUuOx*jM&S7OnnrpyXve$1x5p&a;RLWB(S z0aKbh_ZcNJ{7uD(jAd-n#HdbH{~pyqIMeyL-8heg%g9Y=dxnR++-9nB)kQ8&c4hKi zzeC}VJ0OTB$mz!L`Au1P`VkSZXY(0Ty>`j>URw79ijZtz(M(US zY+Uha&Jf^U8&uCpK^A>Z3GC6?%~ch|KLh_B%p2pX)($jf*F__o2h3Sy&K`S6Hes42 zaq8inO}Zo~yWufxBuV*mJoPWK(T~3jO|ecUd51r4V^o=o0u^C3(CK4Ud~^K|zWn?W z>H%Q4zx9O-Nc%j}2JToB!tHu-@^R1bghGB_LM|wG1M%;b%#*@=;OlxRBrL25;<}9! z=<`_bbA=ODOb6ht)P`a$F%|rl(q}jhar!oA-ECxe);sM*+%w4?&w_7XFu@hkNVP`}i2Jx6WLtB6ruw=Vv50#U1OT2K>0LAdbh6?0wN_E&XS*Dlvf0z#(F*2J7&Z5R!fi=w>fFYWyQ>$7>48k}}_^ z^b}AYE<>wa9GpG^XJBCHU}SC3^}8D25CwC}q}J`F30H^|g5_n+ilqzPi`{~gB4+^_ zY>4eS&rh%Q_J zxQL5j6b>r@0w@u4v`Ymzu>Uj&ya*!QVFlVdM>_NBj{Djn8O+E1&$uIl;qsLqKv^Yz zox#=90sxYR;B$Gc4od1RF9Oke3ZNb3YTb*giMggYn~p4l74%M?lFBhO**I<=g1ldM zb`bs;@Vn;P1^mY1QXB;{&qXJmiH`-FtxL|8o9&z3@T1!{Y?2?-4^Sn6+a@h}zU|Nz zrQ4uT(VZ!IOITH}^3y5(4MI$)t>gozj|l%v3=J>>uq&2$rEpR|4i7 z(D-#_$?yQ^9reFJKmMzvfHS@*xaoN*n&AS+bfv?_I9%yKXFoo5X6|bcH5RATNA$3l z%&sxhdYHfxcTw@1xAZM+8;qyVr|SN(?RJ&oQjz9N;EvMx&*^{%>cEc}p54s|v(*)C;095=`C0*BqtooWw~N|U<$m4W+{MTTZeUdxc^XIHYG zg^REZjF0`~W;aRxubr-VhWV+X%!(YOweoTWlDlK?kpTDt`scSbTQ74HrwmNMe^v8F zomgTqDDYlKQx81$qF<-2az(`#yk3@p4$_gN2m1|owCY}Dmrz(owLNW3Gk9UKj_D5a z9ei$@^VhjGur+Y&s@oV!cz<(0-?{*GbGfbAH$ich-TV-=js~I{!+PgkD z&oyEm6Z;D?@%HOVVs%9B$mNG)$Ew^=;i!_pgAvXI+)mNlw}SXv##DLNDfg~=*ac3mB?FFx zDv7h;A}{`jIAaPS%dIYIT=>!w->7w~7v~WWwD!GVH#(-2`pt_+=o?KQJhd-vDYOdF zLtJh;{JNEbKcIjUnW=gcD=1?%w(kBQ?~bei6~M>x*H#|Gd+`W_#9{`mJcy2i+=flW ztF+{0_(l3e4V=KvwFgN8Bf`oyhGz0};=A6wCw0El$K1$f~A0VBa z?pXV9bLIh#4S9FZCU64-&IJ0L`tHz*EnWb}xIqUN=Y1-OAt9<*fZNd?dAXysNigR; zZXxX^FwoWWZfdMfK7UZM&0kk{kSO%jdlF{sF&C$IS!@F*n3IJG$K zu@5-o29m7wqYW#sJq5)l)bGV7&YS2`vyj{2W6>w@{sbg!rmnkOztqe&Y2IwmyU){IzoV=}BINDgS#HJ&(DyifTh$+Cy; zcDCI{!k<9)^-HuQs5&wL^RRMg9o&5kpp9bqu!B&zlu+2itNtIkQF%#BUSC=@;-dwr zQPt!Fj|RNoy-KPcdxj5(poWGT&e7jJlP3{Z4+?4i^5C=9vT?gseinDy=XcOw24B!| z?7V?bR2l`$=RRo7HJzkRFQTDO*V3z;_!3xYbQESl)6>&)dvYLMR(^Y}5;aae6tz(! zsHCZh5H(dsBo;1%JHPhe)~EGL>%k4WG(btWuKXFkK=l&?#c`U09H>6nwhZ5(i<~8M zolmAN`6xI#C~%yNkA7Q!G62)-HN<1yk^7%+4<8#%_?bmRr_2 z>F2<9cT_VTu>PV29fdQS>IIC6Bx3l4^$~U+P;Sg z{FoHA8(+mu6`26(5~mMH8Z2sMq9C>j=GqL@*D~Q5Kg4V}QAo2yHc7BWMdoQ>VOp^cr9id!6(<)hkmYu%ND#uicK3yK&z&-{vRLWQuunP?I1!%X_~+MiEU;p&~1hkZrHb)K{8 zwrRP70&3h0qTk0iOh0eyxbhM`^D*4sY$hxKbTK$nE3}vs zcQ-|E;JITEM$ny&+zXmmcY@}Gj>gzv64ZB!xguS$t)P39OIKf9<6NY~ybCWfh4Ud5 zIq!MB$xq)NB-`xzp&ov^D6>2V=`p*e8U7-k=c#eHyRQ~?w#D=btec1SB>?XWKt?0n z4Ce)x1e~wZ9+Jdz$0DyYKj1BHPxs0_^h-gjkwxbPJse%vPa3_k~I0^c5Yqiz}zJdx-gOabma+=lHq$g(03ff66hH{I92A5_&k zikKBz@NM1adWYu+KuFv=CF0BC)mbx|_!x6YjJ7_l#gs)%b89!9R%Cn{TWf~?^LA~{ zpVx(y!6ohIA4Mhx!BC;N8$7NZ@jnLfNTjd;aN0YzD;1vx{aiEZ5JkR3!D$QUNFD|D zSlrpe06P1Hs^F=Xy*}qs?c`1-^Z@%c5}^jJ)0;lqtzQubCx5WIiUIX1T*J9%eO^~O zS=)0sxYf|hij>X{T4n+9^dLZy>7*_G5H8b*@>Ja~oI=~QelV|n!w05ot;$Yvn@0w4pbgL z4ebYY`0zTHnNUIXvug3fUB~g6mzKOzm#Ee@fw$l?{|&H+g~mHmb0YHHkz3i*W!qnw zycvAlm=Bi`03B`5#|~fpv7K`6NMuo7&4%)G7oLniG6wEKOTf2dS#f~$Bd-H@L9b=a z2T?OWR#0M-dK_`_9IiBXm-0ijiG}O5;4DvLL8c(SL&zwf5qMbu44p>x9-cV(qr`y# zo3UVn&wIgCLN>Fp#TkBp6x?O7qL2joNTh(bm`l6^Y7jr2F+i)g(ScxF!zW`MeXnW= zV0l?G+HL0hm#LDE!2mX=MpJ=4$4Q@Pwh(ZM3E1JcXj(5dsX8MKuHgVLb9$o{&G=fK zGN>PMRb0RNe6_=_xbz)v$(`BW2q`*%LOE-;DX&!V3tZ)xo?g`QD*r-@E^FMC$wmx3 zx7zH~aKiyO@CF>X+Jfo#jUGQwkZG`gdo?DJIP_TCQllH3v;;c_>!6AP1hmTaN{Saz z_X{!EZt)`Qf%bPYPT(?4*TIkLEG#bzc%)V6{?+oe-)%+8_H)GEOH z$9AX&+dj!Edg^PmSIxXL&&T1K0mjwb7gl(G#4r%2gYK?zh(0UIpjw(vI6t_;rKYM3 zD#fSU8*UhO6iM2ogV2&e0TNuQvV*A4esB!FX5vIvU1{|*i{XqzZ%E1*wK`iT-%;Eg#;LS7+-0r2^l2He8?j;tO(An2*Hd`=ZD~bigDKbvoIyq4~2lL{l^7;vf z0T2ae#RtN_3!2NmlKYVwKjy?W-#2Gx7Tpot_86G~&|zG`Lhg!Fdtf6K9E~$fJhyyD zp*zTrZO@FAM31ySuzt1)P(i2ME)Xuk;#=Sx_oz5N_X5nRW=IqyS03JmVB?mb&wQ}2 zQ2qXO^PlChNM_h@5r$u^Tm7_l7r;t;&3K-!?AS$^TL7iK*JQuqegnBJ_H{n@qrHq3 zaVuAUjQ-Gk)VU*VHpup6ybNE8Q%KQ6*#h+A@-U>~$&-iDYa;TR`Mj><)gEm?*<5IW zP^Grf{+f2!8|=dZ*Qa%=*ky@@_`#}LbQBe?ak4qf=Eco0Biz1m@s_DXft@}_x& zIN9=pv$bJMVycKe`E2!&*ze>{!+r+=3Oikzu{sacI&@ z3NrweC9ZNA&;hw*2dHMMjnCWB07qKt#=t0eRrD@g*aeO$-vgq0TiBU4(l=u5bn)Qk zsM*b7GFdTSJIZ=<9KT(T)#sng9ZPm)^f_c1BB$Du#ynm<(UGVXzFo5qWSo9v9j*)aUG5*q z{iW@280eJV?WcU*=F?FpAcn?HQ`MDXsNw|Le&Y0x^KQS~^gZ%Rix$GO$6Yr7NI0Ny zB*c(9{Ep$~&^uUah0=Syb2A@ov8jQZZsJ-J(HtV$NS;oEvu61oZT)@hu@>> zOxtG0`<@JK8=Q=u?{54$KBGG|wlHIQ#vxHb8x;7{T6}C0YYx()M#)l4BB zKE0GTaOp_cCjDeGYutIauk-~k9rE{r69tt8&66fu%Z^@N>`U9wD3oV3%8?>5=%ZY| z@C|HVUx`Qa^*$W)bQBjb4$8}9FvV?efEe9=9oA6U025<;Zm}1XxNchfUg81~I03-n z3qkT<@IUy0-awR@h+qG#G-UK%`lYC38;)CNBqzr~PdR9`1U+6}Y@kxnW+lk}p7;{V@9i}1m>E6ufCKDbfeP?v?MGRBM_lbwTJAFXMnWk7&jwNr3 z-Rvo!c(B#iM{XUnvxg;=$!BqH>VBV4qB5Rr5NsMEY7T&jZH5hkQ}3^*J7i-ze8VLfe4Pi6hd#>OB%fBh zr`x)tm<`+>uMpv46$sK`+@W7^NU7v=??I|w^8SL}8n5Ju7zD=6c|cLnYLvs%mf@#;Ifl&_wOD+wDo7kAL{^nZ<`Dk0uw>>{#_4gc+h< z6bt@5q5C|MzoV)r_oeE5cqjLCJf z>JLo3NIiq^Un7$Z8Skgej1t6Ha4Hd78hNH)%uc>#M-9I zb!FeZ4(#T5mD#1b&=`KG>SH6Fc4EcHfP^lUg~k@uyM;bqdxkmP^47v6!c+Sd8CD=EW+I*bnf%H6A-<4ekL+ssYKW~SBW9O{O%re zXKQZB@%GrM*{++}H;$=N7+o4Hk56Z2VYw?Xz#>k2!m?2Jn*3u+{`0L{)RG3gcphj@ z!=*IG;gWORZ2KiuOz)>|xt(nwD*$@imG+#e2l&)GV#{4@cFKo zD>gx92C&B#+*YY8TViR-9)o2isGP}%EbEfIUOCcfY>!%H#+DaZ=qz8KQr%X!-3!xA zG5YT>ijIFxzEarxT%l{nyVG~On{-_UCDf?fva|V;G(K-v3p6`@!zY53{%p8S%m!-j zVkwt?i3$b7mRC1!ZROM*Stt&1qzp*AL~-dzy8+{5v1if1+ZNtWGG1`a#TdR zdk6!N?oNZ0?gmLgYCyW)^Tq3Z27m8=bFY4!GbeVez4qE04zm>&RWpS|6pzi>Ma`>1 z>z9`;4<}Z9If@e&aBta;?^|EC#>^~JE}~rzZx<%BFH~X=!>77^5ur50X?OCefU_+9 zW~qRNxXEp^Hd2{*Vs@==w|R?zon3i4d(j!#9R2QwID;hymWKqmbZUQJx-#$!JOE6a?K>(C?pg$<_ZnX+41wnJFgB$EMo#zMQMl5=%fG`@D~0aO8yxOvQbQ?4@;4V0t5pqNVdoZn&ZPD$mzS-AgC6= zj!am$nH2Px-#(`Boe4K;K>m!_*_~67P`wp&g$Jf_&Gp_!m5!JkF*9AGCbnr~MpU-l znQZDgd>?~r_Cp>$4O)$fN&=>q?Z|z6BZ^Zc z0uJx?OzIXrg9ahd=Om`}`nLs)lOnsR@9p;dgNKs1$m(2`l`r)di~<344>{4cMTOaxD%FX{mt?J3MYn(CRAD&>Lo6YtK473}r{9=Dj7 zQvb~nSAM+fNwIz#kJ{(7Oe~ZN&E@3BB9X-|vOUO%9hEAPNaRPpJ61kE$m+kv5b=PJGrf6O(oOg~vmMZ3|b1 z^DK|=mh-t?t^N3(kfAYKJHBqI^@j&+&KQ|IQOdlNT!ZYaKht&&QV&xYnZ3WN-5y$? zr@)ISH4^u4mmdAelkDeDy&h+S^xyGBDaf=F(&}ljYwJ7OZ~M(7C${G~S*}%cX|R3p za9T-A^TkYEoq4=$2E&Sk=L%rB#kZ2ZYWXUj zRfBu~3pY}prjB2OuIV(fb|7&5(d>FSFG?9mdH6MHHvuRHV5`~0lBd*I&@1S^?SGiU z2>hJCqL~}o4ujMWVS;Pg>{jY`W`Y6`B9M-8QGR;kLW^#TN}%Cc@F^rLCc7)tLGFfp+oP!?!7ELZBDUsoO+#_@ z+8DKA+A38Z8xwW=y}mPp$0+#N-VfWhTS597!w(rurfFNuMR{PIghSRPK_=n5N$^Ze z9uBLJPqW6Se3U)h8Sl7&i2dOh$552wS)RB^MLdAHlne{!nirO_JDiz7*(;^5~EK7N> z9d#5dZWKP41!hJv`?eVWH~MSe9)R|D{zbgEfV; z>E{^~7OavHSDpH=|I2+%z3;KVcw#O>;rz(`%X`o(4Dp_GeQO_W`3xiYl6*t*goBXR z)Qmf51uGGCUcTwS822kD-eKyO(P4v{{D+Hjw=(d6=OIa*PwV?i3#io01m$K*HoJOd zcUjy4hLwVk$h&zx!25*6edddv0KVbe6%TTR0FHL*z1rZBw9NG9#>i*p2?760yP^F1 zgI4|Ulf~SSAOG9iYt0|)8|iX|61sa*>6bT%j3(=5Y|W?fB<(O;+4x*m$oU<#dNllF zY*qra%C?g=?lD|u_I7Owv(S8V4gD9+mAq#PO_4`{tl?(@MxCJzG6N z$l$`C!g^a0_T|T1Xq=ZxgDJgR{D;iNCtIhMOZ1@!gET9a{a!*rj7&J^&yqr#US1M| zj+lY%Cp@8-VG&_24W75^^hK}t~x|AmyFZg;yC~Z zAu6#VC%?rcEU&p+a(S7LwyKXUCI|e4=}kRi*Bd6iNDGf1vo&YXLoEc4w7iWfI1QPr}daX^y9}7!+^3?E+DCibvGSq%@gf?T+Eq zChti*#+&?iP`@b5e~qV83K3nU%<9zUmim;>jc>}v-4K-U%$%$83HG&*zHkZYqLYxH z=d1uefx~7CV%goglj*SC{ZHNZ1tn{@VQRN7^MlBPq$@CD3&$T#G99|MeR6OyxfRPG zG-wHa4>LZ2W`Lt7RH&eev7+5behcX6yI^60ulG>4>OKADgvf!pn%E4qC<3mNd=@{mS}7(9R-^4%m$`lWm63z zkc~F9PDJzBK4Z>edQ(*oJQ$9)M+Ump2e$?XGt-6sfe*n*5W0bIBTwHmGL-aX_`3@ zsTK4(*u40@nyk;GOtgANGBR)4iq@6~kq{BFBi$}*;@)ZqUbn*3d;r(1QNHDUdN|b= zn77->BUygZT)lX)!O)2Lr8g^Wtu`jAhr9?{+r;MOXyrqgCc*weO~)Y|R0t$@cn-LA$S%%bL`8fd#y8&*CO z7UsWGW{ASD9_~@A{_s?WstLw&pgFTaL;>Y{H%yoyQHT%HDPxhn$ZW;!^Bn5Yx%Ces z%z!tHW7qrsLbNxqNQYDII}XQ_vxexlJoqb{uN=;_7Sj_!jElf)okOF$M^DfHs6 zEG~o4RPL$r3$*OvIs7%I>Mu=dhA(p#>Ob@x=5}5WCzKb!fCUTDJ6cBUzCEeo1&ByX zEK=lvRMUH>zGcYW;9bF|c;SASYSSYpHw+2=3E150_5nrv3wfvadWm`cI|pS@r}u%Q z&H|T)7v_j&Ja^|pS#?E=45ys66w_~l#(!wOf*RN~r#T){$msl=E{I=KAAV1rj^&hV z96Y1MNp?9C+dNwa2TEv9RswuyCJ*z*Q6w!y6;1ReNzI0)9syD>~%b{7%t4&wVjzts$xuYW9M=C`++oRV>Kdd(ocvZylE&=Esq$xSmrl;f_gcN3L zmRjvk%Bx>93KpfV7EmG_a4dC4ygC+y?xlnQdqS3BWBtDkf|e8N2{vvx@9pnyuHPm_ zG7Ro*k7+}WsKB`)O?C~y62N_oJy+5tS~xgfScSrEkNt>R+Vx9_6)j};$tCR=ckdfNZ1t`(NRe#%-qf=?v2^}m#bja zuCLQ)8|%VC+ofx+_u*n41ca+?_wY^?Cc%-SF8(fMt?+E=LD z38+Gu5Q+{;mXyoil-piFbd)5F+QT1`hkI)BvrKdqTyXs_)1?>M%%xxp#9e_7*5sZ4 z%PfCahN3`zkTKSJW@uh{3OGJo{NeAcFj*6~wo`_LLR@|O%bLd`8k3;tS&Lapcak>> z-8<05Nm=t#eOZvdyZoOvq#L3kk*<{@1HElDiBbA8&a?o%(OX%Ac=Rq?#hgBP6_s z{<~)-fXVbS*XWcH&^mI{mA3&3J)s;3D-sQET0Nqu%ZMUeaIQDA4m3NDKb+F{Qs!Y1 zy+x=AjvM~N8sF3gZU{CJWS2?F1!pBB_JOP-U6lqMg$D<%am}_-o;dT?L$nKVm$qwC zqg5swCL2;8&YqX>OIhp|xT}x!f5P4QGB~7(2v`t4*khF{=O;d6Pyh{R=)b&eSw;4y zGJGyi|G1Bvt=FgeJOtLC$N2kD;S<0;%;vn?KuaW3-ojLy%^1Cn8T5K5 ziBr?sUUS%jC2`42MYG9u*j%sm!{&ld0l<*#b{3Ei0`PqXw8y4_GHpUz)H~S9tob~9 zhhHA1JccefosBNq?}0A34Y=TYxp~WIEBROicS(qnTarl%Sk#P9Fxc$b)}Ez-+OPYh zQ|R`LHXkx93hwnWt7AaS&v!UTvIM`7N%!oJi{IQ4FpC2%622E- zu-oL}oE?W_H_%Enr*8>Npb+cu30g)!bA&(U2&;o)K@>$Y`+9YW9J^=bn>DaHTQ9^< z-d_*os~!$*vbaUVk1;aRncJ5-uQAJjv{P6nx(`fXLgF zD24;g?t!iTmHmmF)1dAD7J;cv zcc_x+%3=Gj@b;R5(n&w2y{zm&a4eh9hzwkL+7!%2?DsS~$rR?`c1D&Fc{z@whV&?VtgI6Dz1Pu1vr=XArk`NfJxZUIB zEsu6jw;Y74Alqcji+y?uZ8YMH9SoV?5E339qH0TGLLhFk%o=6tlVYoK%00xP`)h1r zs2I4?{H&RgXr=?Kd=z$GI0w~19&I9s-3_AilnVg#`2m4Gmml%~_O%A#5RYj?HtT?F z8{lfnPX$!@Zp1_en$zFz8rLi;xU4M$Y-2-qRW{m7fsR2F0%lY3r+<5*(@g*~%G!y! zTf`}PL%&M1F9nU33bb%tF1ZGVhb-a^)u!?ae|iJJ%GbDK#T5+IC)Tc{7!b-eRNx*9 z1GgaBNsBt&fK^ae0azzYzvchJrGG#4L6Lb7;xf8u1K8hrmi3n$WN%nho!fS7n;`3> z29p*5uXQO`d+ru!Atk2oO304g`n)xcOaAq=vol{Gaq&t>Lw8sPW{BZ)1FYic_;=~; zW;hKWovn)c0+qD%zy&z_^-BMn|z& zAAlcy|JM!a|NiygAC#5AkLKnMLNnFy@0e-E#j}LR;tEfnC(X95)$RKC-{~4xoC~`Q ziEdb8LotPF{X05%3OlS*XOu!G=Q!g%`w%_wWA$zrwHre}+>*8k5$U+BtG`H_dc{3> zNx&gHbQk_*n5h&j&Yei9Se$7qQD8(Y#@$b485Y0>_*alijlC>uZ=@1~T`zE7Jb#|< z$Y*@O8vyBU9%FesGe6VESJwc7Rap-lf&VdntkQ8nRdy&R!)-xP>J-uiY#f&}w-)>g zV8WXJGGTBLjI^B4h>-hfw*#VT9w~KLSr~|iWHqPd3*85LA&syy%K`zpAivAn%2M4j>dm2w8YrwHAs>vfnbt16@N8rTyn{CZ7X=KQ zHbh!Z!QUVv*QqXb!}GV@jWPBGKsuPF&V;SZDJO+PeYO=1L<$;U(MjUdXvP@>t4&Ye z!`%7!qF^}4ZKW$Ur^oZO3Dk0yvh{7MfH|Z(IDO5lJPgZE>%#B;YrzIiEf_d#Ow3v* z$HNx~LkPG=O4;~0FOC+{`jM9?jr8nfDbxW4xrIj zKKofF4iDS;hO-ks{$)hMFg$uxKA3;|mtPINEkrzSpsXf!`g}7zAFYD9wZF?mzR>Y_ zK(u%UHV*jK4W@ihBT&m*Z8fnk1gD+}%i7n>p;A|Q{1UuP{Z-gi2A0{jj(u&TRr?0P z&p7-IS$iIL2Ld@Mxtk#WaT@6S`(cZ?D^$tGZWHZMhgtw&ZML#HLuQpbhRt8ky+58Y zx!4-mV-O8m-&_u|KmQJjQ=+gerM-+nA@Veeeh(zFU0pNn&c*U`!h~1kl$vD<$Ou0g zBQJbPINL^#buG5&xqRik%>0?jMMy1BFqt2dfA$(4gu$*eLAvHSzVEuKnI+lxW+)%T zmy-Gi`g$*od930wz4mQ(8k*|tS=*`obMArjU-omBzQ`jvSqyI9Zg4%KH-ZsMTLFN9 zB)6D-d^j#tZBdxmJkHf?UvZoxoDy5fOjl~3mvDwtD;;PblmjegWi4sBw#t5c(kul_k&FXN zQEh&t@ko(WPA%aoyP`%cAx0`G1E%=3ziP%*w;9eQJGT3M>o|2J{~i)wU42(Mz+^dQ zhc$7lw>QL5qj@?c3DJChHK5qT08c6X{o<{Bium_~$0yCcy6p}jb{5&svWdZaSu~ z!@yGwq-$wKoDNbm4SfE3xUt;CYG7dY3yv_qMwX`4O#lDa-;{tVijtrfE!w#$z@e2e z>G$`>*-Jb0#=S<`8+HIj!Qv%Fh%Lb9TfasA>|Ode_SV7_)AE<}#kPnH)&paCI6y)=hbtf(xenysEOZ+gps#v1WTEw>xmyyD%QR(#27giqy7T@ zE&x-v0}jVt&j1_ka1`VbhO|ms$PR4`W_(j?bN@l@Qbw;GS@O@=nM7g3eozA}>k4xN zNcPFDBQkn{2w7)W?|l~Dg)vm~dhgB6%ZS4IslwoLMl-2W$O^!ZFy$JANz&NY8x(U?J&iul_>UBEGIHlwPA7`A_`N^>#<63WPl&h@P9Nk4u_N~}IGjxXxA z*}B_5s@n8g+={g0!4Gg}G^XZ1O)&D}g93qyzNJ0=>V)WipKr(5WQl}WN0EXMpay7d z6n#DQAX_B)_vMb-iv$d{S~74na&)+-2LhEYiyQv$dw(y$?Akh$8d)Y4dP&Qk&eo%k znZJVB^h!j9xr$o#&eqRqU)^HX{Si;D&@cGXe(*F*khnb+@cK0PaOzt~FtTKEFW1w= zyPD_mb#7_coXWY9?WY+A26S*L4%iI4Vt9<=fKbSQab1Q*FtP_5pl-#tKcnMokb#}2 zo61gbE8s_KFg^!R3_gKu>V+|Zq(=d8CVtdhKQOS9-rruk_C^J~Yhb`IV+!KKBGiP^ z_w1qV7|=FD0^XOe#XcgwoWy?8 z^l%CW%b~l>PeRo?*BftVuc7(~&9XyR{yr|>cv{z7j1XAkpYtra#&;Xgjfis_vxDF+ zT1>TTd*7yaCaS4H#i=sf!zwliw9NbbwlqGq>-vekrXmUpaYIw+CPFKGxK4 zBn|8qB&dEpn8;^uWsrotjDihfLrRC&dMRV4c>de-dbMyO+lTZf9i)pC?!Jc31AcW2 z7|iPgb42z6n%bUq^&IB_6yHS!Qi97bI){p_fC*D?ASPbL4eyNidUlX$Na| zDr(ZX^)6tr7sERI3KpTz=fYh)gBnbMaJjk|-9D`~vvw#%{fsML5=GSxzq9Ep^Kn3Kf zQ`DiBxIx_Cu`g>$53*er2rZb$)jHg~B@&VCQDo7%?zX+#p;Q$P!Ji%U`Bq(>(M1)m zkkmy9A!DA0DJ{_KpQmR@)oy0I+G)|)B1Sa#=WYLa&r81vNmYqMX4 zg6**ojyT$t)tp{?1KcP*y&9gRvH^eo#q98pSVnsKc7RTEyva#_5S8t9VJ^z1m1RoX zH0y)e!hJ(mkz`m5``VkQ<7|5}a;o5mOJ=~Oiyj6@5Eu z^0OdjjU;-&hqi4-Kn|G8$Jb)2Q zuBc;9^pBo+kmzyB)&6Apd-T`TBXITPHOs+i|M%8_#(+9qx_RUUsknM^>Cu_=vK;?O z_s!R?{i#RYx;^m`h4|RMaC%^m1T;hs(pp#G~iT0un;6t#!%gW zMu8h(Q#q(Fpd-t7qV!JfK|Edetvy0i~4ANBi=6ary1AZ5A6tg zXnM4GXe$f7hY)VT{=fqRQyrwq>?XpvW<7r7kWVKLUj4fC;W0!fK~!Lk6*esKIiQ$( zY6YD;b3V;~`Z8nZ@nPda5{*VHx521BM8yQ?%}cmVRRDJ%Vwey^kqIFEQBUy*|2vA` z9~2D1A#q>eK{tmDguB}Xeg^dHKR@FXmvA-Z#QbVHaqZpTtR}eGPx8SZsxWcw*qf+zS#4@_!@p>xIQ2S6__00HvCM#67gXp@@#wxlj*!Ag zGcjQqZWkfN1NGOi``BnY5T`5pSP8CjW(3(8EjqkO5p$m#c`prcfKosM?tn`ZN3FqD z+&pqncYsnSN5qS9KI`5920QC%vB8dL3s;QDHspLdgW(RUf|qX%rOq^%fL)CBeJ=lC0WlAV5r6yz=*pIy z`a0wYXeX6j)=3PZ76|SmD-bD@y%BwO(6}20Sn^-qm1-}r@B>E(5KHN`v8(nEs4)Qh zB3>;?Z&(!neFW#Gz`(sdk|B{FCUVDFtYTpF1zT78aAh}LbK4f+w@#v> zL4RZrq58;pPxE4XA~k}Pf}b>VU|6g_(7~DH>~t6K%p366IpiBM%Pi4CqL1(EU-~vJ z@g-zn8Hr70s_3AUc!6M8wUv^`$!1LB$nk(B20I`(WU}Y8X;7qBWMnMOklj82amNKp zKCY9U$5q>HHcpAl-&%lA1tgX7J({dp?dFzE@blH*)cfDO%#Vv==_#0dXs+c95t7QB zY5pQ4VPvuZ*t0U}4Edm*0EWvS_RH_y7&Q=5IyQP5$g5kyC$gS)8MUgG>u2?{rd!qC zz)S8M8yn1oXh>wpWuQq|a%$Vi2|0{;+k;<&+T@urH7%>Yy>FVXenPTvMqb!Ip!QZ; z@2^Xu)9j4-ljP`CFmnHinj3vPpUMU9GyVeHrO@BC)#NF(&5|st7xQfb-n=EnEH`YZ zhZ>-+9jB3i#ALx8uiZ+-Q1J&g3WVPakS1eWQkaG~1svAta!v(at)?|=)@bJ8xjm5)P`hR7J4%mUrm@ zuD?rOSN{__soIJDSXoXvZo;6y;A!OV=_Ne^m_lZLq+NtrLO=s95t(Zx-5 zZr=-cn@4#XJL@b}#>z7oG^JaHbwC1km!m7(gGS@nd*tDiCCo+W?hPeQ6L(|lknviV zCt1sFRy^(|vh|%Qzp3R$;`&a7Fb(&;_M{+yy#US|T&kNr-cC{Z(%ZkcD+7Ps&or}@ z`Z}7K*q~3LX0RcOlCQy(YN3lxuFDnl+L(_*HLVR@U#osX1xVG8*KPxuTsRSC?qnB% z@QP|(CXHF{>yZkRkIZ{FZYKHL+$7CvRlm@Z3`K3_z^&_419Ex+D<+0LeG|;5{&ZGp;W*Vmscf@sYdFSM-PlTFr-h;L`n_R+Atq#-fYEqMMmfXSF_H}!2I;}ieUlb&G> zsd4C3Q2D}u|*TdHt~Yl{HepzS9lV#vSM#{ukl>K0O6}L^N!H-3je6VD*Vl{q$`#a5WI9 z9Oc67!yLPXfsdF^ODi{cx)3*dwboWcDD=>K*JoS|)iD0W&dAjtT=dJ=!0K;;jtXvl zVhJI_f~&^eoHSsJig8d=*@JgJ~87 zXgxR6cF7nObX(GhwJPrH2a^Tq1h@uicq0;c4u+MzojCESLU|E@xbGVL8d)7xlD3B= z%Q|j-!+p8@pO+SmOz^N&G~t`mqwaf9pQ>;1*<1nrfI%mWbjYQq*A zKSD0(y~jb)0y~%u%U}TVWl-#4yeV*ZqN`j+j>_gw);Q4lP|8bfq*!=Vuvbpt~;o?39e)QKZu3gMgLO!i={yzGf!{K)>o%i(Q)n# zR3b5s_}ez~69FPe?`J7AiYmep8>B{!!Uq!5dp|3CiDFHGbnbn9S#K@CZ`-i4yKysK zs291}@PU89;$jnxhW;lo8WM(4YO|W1=NIP z7^qQl&k9L5oCnADQG2TL!`{e4eRi-w?%B46tmp$kCQi*Q_}li;V3jLB#>-@tK-*45 z>UBV7R>bqA0=lR9P^-%+{<*4_69lHu7*=oYBg%OymWsr4*)2`30(SJjyg1>_CzAx= zr!*yv)Jl*g#|KsIgoxZp76k1Kwlnh+rlp~CoEjdGV39vYE6X`i@5G&FL{Eknu@MpP zLGhv9bc&*#@A6mPGb*o;1t14@<}WfLNOks%dQr1W%fPo5(&c|FqO-2~AnJu8%9@d{ zwNN_rC68&^9B>v7kXI>Gk?iJ?nVL76^9$!3C!bH=_S;EBzLAiZ8xU&Kakm)#rL>75 zqW~hYV8uvz$=_4j#xNn1ExeLiLHXm&ecT)5am5(i;KJ9cHU;p#P+7!=pD|#+3ObOJB?s%Yob=B1=Ya+y{QZ?e4W-9}x)92~ z!f;2-7!yS6xzfun%M^Zb!^1{Igx^y7Foj-5SW3rV{8l$;1%1tJUPKIoH{~!^o66LW zH$?XH=Bn*S+%w8bz0e#*^oR=RMRCm%9uWa0$p8HbV2y=L3gl3y=T6hpDhMLdc@{u7 zOVi3p57J>!q@BM+iPwGrqx4E~Q>D0-2Ci1M6X8%?HkEqF3ZQKY*_yI~2tp%du`TQ# zVq?h#bPL|OYbO5fQnY;YX}!QrURubcWHRUSbmy;Pz**7Wm;hDYvr7{JOPejq z1+|p_Qgy0Sn2U%2KEuOwgU9KjO^TW{Pw--6AaJadF6()u4J`quJlDclh0Rba|0ceZ zf-bmAQp2DWHU%i6LXJj<+VW~gs6_YI+_TC$+@j}8K^_UZ75(ZwrW z(g*KbmK?ZMI~LX@B+@qG)LV;yIoyC@|0@ae-wrl^fIHB?CqAQ+dKU=4lB2Fp3MF>` z@R5BEr-SKOwc7ztdx}n-?2ISOH0c%6w5pABaI4rCdt%K>Q1O=uTaWccB)>9JbV5AK z3Ep&_nu}@!P!L1Duvzx$r2{WLfwFXzp~FjnE|;ErR#XTw zhd8&U!rL&JOO?^ywWM!-`b{Asn`(nFP@a5$;~f>a+8d9|dG-3-=bg!`JkLqZ6GULc9c$UxQ6nB`h1dl<5|RDS}`tP{ga~k z_+~SYNSPB9zVS6D!mdOk;Bp;)KRA%SwajSlTZU4!3Yc! zGK5oDS2~O87g3A5=I1A_u8asqny6N&+?$#s(V_$7R4_OajOAKs*+|Z-@_5`)6sem7 z2m^gm+siZNjr>&sXH?uyZ(ct>sCckFnnAyv`d47yn_CeML!$uKnU&SoaFMP?wKkw8 z2J=EDJ!ke^5BdWPRctIKdLrc!n9+{W(@Xt2n8cy`J{Z=Mf;TveNJg^uJJ&KdGllv) z>)(_RmzxzCf7T=?PyPq7j9bmFvE^c?CcRA%c25l8iKX~DHB;O(ebOUBe?O8i=>1mg z+oPqHDNkU65fS&UUY6yt^dbO-=9>tofRSlEohmM`6E^Z`;XI85ZL>;IVutohfN}nt z4q$HGYL8=(2b$$-6lhC*g9Doi@E8z0pG8<4a+fTESk#=2>9p+tB$ToWPN5w&KRXTM zfo&44yA65F%{wEFkri&kXT@?ML!<|+^JbL8ndp$1N}-nKP|q7w-*KrN84=nz+Cu1| zV6BtWn8#zA;xbe7=BEV?NquZu`}Rq1e-YDwsxN`d8o5UEHCzE_wL*cAp!z+G(#x83 z3mzg#wD6Q6*rd}|_z&|<`L+fxB39Oe^XXg?95U;lB`m=7q%dLh%k}TkuA7;a-1&3}R>e_`7x`NEIt?DM%a5ybyc5tuck!4$r!;XU3`I?=A zuHm}~X(C8N^+HMMu*^yiW zOwV&v@@Qqh-wQQ|)_39K>oe<1g?S(M&)gZVXZfl|mVN}#M_(vYbRF%Np5B=(r1oMn z>l)@6B45d!N)DcWGaU$b6Za$M-Rh#Ul|y9A-fH8ZW0~c$Pmm&JtlkX!{_wT_>Lsj5 z@1_yx1i?Z9B*F-Kt-LqR?O?TurHlp680NdxtJuCVM&g7nqYKxa2L2xrAZjH`Av*(E zMd93_hzs`KO#>+IRw0;lFOQ>LP{`@EoJNG|cCqxQtns2^*PWj&XK)&`h^z;!<@eQv zuY&3;FjhS+xn4)8z{qP>#ySXI~4nf`DzxK+>tMF|BzmR_r`&gIggZL&r+k5sQ z0XHzK+sp~6NvO^#$EF;FKA`RQ`ef?s=vC&8IMAFhR-eoi!OqQ)#aVL{RHJQ^NXg3K zx32miI8dya^*l<#X7c*U_r^Ip)N>Vw z`P?6G_HbAiSsNTwojL#!iU_U`5`5=Wqo?s8abdo<^h8|P;I&}RD`*Yb$CD? z-2^Cu_h*?~MsSx!(x}tpTLD^~VaEFR^6C$P`EKiMe~fM!KW5f9t@7WD=*{iXRZ)J4 z$CwXymv$H{Oy!u^jCTT7IwxSXbX(jcfJKvJm+o;)Uzycn(1t%g4{QP7b!uH5>f9Ub zb@CQsYrp3tKlX~~lqU=@9F^ZpK#-V}iI~`p6>B&cOe~EwdpJ{4+RXl{h(7>f`;19x zoaOd>gf5#F0gnU9mM4+G3=n7{9KFGz08IxrW-?ymb zocy3jnD)&he^YXlYd!uWe%9%D<)x>atgNmIK(^h>rD#=5Vn>womruOE7B-2nKKDkfl*;GLk-=DPc4IbLTE z+zzr^6XYjV64}6S^_K?Tv^q>_ws6L)yLnk$}9^ zeFSL6Y3Q6NWImn(J}*d()jx@c5>G|aq|D)Wp<*EgwO)s{3~DDKiAgev5+m`i6ffRN zQSa%;AIx2(foRH~u`D$PZbE#M?ulueSDy)MaVgaesXOI7MZY26b6mZWa ze;BymAB(>1*z|fJ`Ed8j<7EzCJc=oD=tlQVm z((mCG8B9<#*x2%H_B zFRy^H`nGJD^4lID!rF8=6OuwJOi=|$p*fv}Hu?i<6N5lAkzc9rH~D&?F*C7sa0&aZ zpK3ZIl$20yg=18$agIe`UNn{#m*L-TF(P&w>`qxI=dYkpNUzzsk^BbuS+PMs!J^)3 zsy+U};gQJUT})6E1$7ljo@1@jDrm znv=ynBb3cudI9j9A@x>T2jMDr)x_O)_lPSQ$6)X@$H908SL5`5CwNJ&W z7hA=zF)BvH&H+-pd7d%$y-y3lHF0g#`JM zAh)nRhp<0s7qq$lhO*{K#f}3>{>HD*m^URSY=-CQQ%Fs$b+^$cghQzb7MB#jF5l0# zqq9m8xL6*`NrtGQj27JGIWzFL`Xuy06znm%8Pfn2L67@Oq~GO@H%V=U$?vd&Y+2gn z`8$j9dcW?B-H}g@$sCM?u7~j4+Qr@rA4wUki0}X>6zwpfC}RxPf#fR-GWp*T1C2}t1Zr{Eb&cdo z!?LA)^hbzr$AkwuC>uuebnz(x?;Macq~GU#X_97Qk{$m7p@bkfTmo|_is)<)CcZa+ z7L3zz2Zi6CQT?%|dW9Nda)z_*fXBg&37$N`n)^{@xucs2w=Xv_ViV^>&Ao%_PlJ1p z+WDL=@KXG}M12ITBUYU%2k1iCrgN}|^}pDOADGF`cmn*_Zf31mqs zN8@LjtTk^!j3Gwe;Zdl;quOBFFg@oV9ltVTq*UZGx*W;rLM3BB(B;@K#)ZDGuF>@u z*ZA;#e=1j$|Iq??jVxbhy?zy7IXH;mfqBq__sFnAi??w{$mP+mTMYZd8yum(452o% zjpUD)NN(M}dxroND+^QZEIHY_J^ft8iEmLO)8^oeSb3@1!e12#QaDhsk`CXof@~OQ zB9EEB-4LyGE^9_vb;zExqtjB0h^<{VibDTPz;$YJ5XxA5Vl+fguR&59{~dh?+iftf zd~I*+6EF_KJ90_D69T@!Ktv<5OxWa+$GR2yOjVu+ej=M3jk)b3bAcm>@mAFy)oFc2M+2U6iCwFfhboWc@1A6vh4~uG(e6{0K)>TU; zg?UBUht-kpAhv-gBJ^ z?>BZRSing)CuwyZG-lbE^h|}PuJjGbc&tpRN8cy5cnR{qZEjQJUf}en4q?Jp?Ix;O zhcC-4->A{Gqf(8ttdM$bMJ2?1B6o>Gl91V3TyYeD7-B5VV2i#c=bQOB>O%KB}1KTa;rj=p$i>Y zV@2M#Kzki$y4}HqB{$yFD|`&+^b)dpx<^-&)$A4J%&s;2fRqR6eN>$aU~(&=BF(2J zh32V2aI(|v;_{OZcQ3T_Wv~XM{C5Zcd~hIMH(N%w+&B}ZHDdqi$0Q>GGQA-4mPY^$ zl$jW79RkslLY0EFJg21he5{vYqdAwtUf=y7@|wc^wIb!rFl-E>_6=(bZ95_c6qTxf zy_-lE{Q``7C0In1{#<%AX<^2+A!coA;8~mmL6$SL>PI z-!6!s(6r@=oFW=@_Z&#NTW&trQoa>_ti8YlXb2lj3L>Z<@kUQi`>l6-fq>Nd zE0wxJ<;8zHZJ;wgk6~JZ3 zm5+zz0c5s}E*Xf-_uxcsZS(5Q!HQbkKMPX!hi=N;Y3J)CM4isNLk}=OFAAWA>xe>a zD1YL0Fz@`t?jWq(RKpzvfsYl=nE@o_DI$74&Gm>VV0wPmXg*}ZjZ5UHD_= z-fT$ua7>ID(jgn8KQ9yEc>Mg~ioBkXYrAH!oJN{{-6@(5-4KCGpumclH%MhUlc#|a z42a{r<|wax10Dx2HrHItT9?8{V3h_yV% z8To>7p7L{U&6iElp)jB-mVDg?;*{rLgx}mQzpHU>0U~9JY-|Ss`aw<*P{9Dj3>MoN zRh{}JK6wUErFXnK`G0KE28}*Y15#GJ`QJMNR!$BAVXJ=@U7S|4(IQCQh!cSFS|ep} z%eHs7)N=glyni@)tV>H6n9@|SSQr&!5cvSIIV4b^Qk<9Q&z8@r@r3@{C~t6}#GdCs zp*3Z)Nk#D1o1-+NDF3V8d#1u^cLDt$6Hh&oD_gIslqCtME9E}2_H5Y!Fc%T%HMgu| z$WE&=Mw*+Gp%qS7W*MNj(^CED$^2A2!%Rf6vE$Vrk>EF+z%p=M+DccaEGZ%z|~dMB152`LCReOglL0|B4ALj;A7$WWoYOHA?_~*wR0ySyg6?`Gg-rd z%ME`}=1AiknSNDzi(jMIBg&Z^=*=2=Iw6}jM@vj3#X4rL*Cwfflq2soaudfb{2xRU zCdmG#-=(e;Jk^l6#BLE15RnJk94Ng<)WBRGMqb`g8m^hM4ARpFwZQ@M-V<(!w)5YW zz`VBm#faN)!%qk}8J*{rpqvM=wp{Ig(lz%l`@6|IHv%BZ7@1CD>Yu;n%OMudaQ{fJ zRtlkbbj)ic6y-F3m2oWLPM1_;G$Smx7LzY7yAh0Hs{7cZPI(8<^i}=451*dsMjSz+pKH`dEoW}SRM_0Q!{J-wOVJhNO|Yu958MJD90Hw9cnjd>J~IZq}u~I zxvvuhd%Gu&=JeCjK#W70XGfROs*^+;(p0e?8X`zOnqPb&Etw4btY-Ml_6_skkjZGr zlCM3h(dyA-(eyhpJH=l!CkB2xtyFTd%tflT6==$N>YYE8 zJgA#w*;478w{)ulwJ0cZ-HcK{`B-*&8zd;ARivC$whgTv5ZtXqVy1-u*ln(E{)0^# zfBd^h4=Xlqad-Rll2oRV@B@mL!3}1!@to8(582fi@^wJR(N<46Ov)b{)9!0;ism`p$H*8lh#SJ z3VaBlO7Ua%#e`FBszxA4>pgMKgVRwhcPNX$RWg4UOG4OBV6 znDE7;J3`-I)iWrJf%bh~SV&&<=0FxP!yso_zh;U@$u7V-`<{`_EWiO178Ipig)YVk z#eIhAfd@lBBCeK8Ox=K`elW8*hz1f#KcAeQAAs#Pd<;NAP!ahmIXuG{nXX@so;#hc zgE16D+%O`o%>SMku(}dc-wg>+o5{~ufo7F|M6nDQCJEry68|;NiJsjk!2SoaP)|0j zAG{l+W5M4l=Cpq0IWRv~V!~%Ztwy+(i_Ff_w>W0cd3ubiOKH zSg-$`r=8D-rQ+3JX9wU^&?o2i;xmxQ_!GH+N{RFo0_a5Ide@0IrgP2BW)&7v!>8;J zS*lJAfKx64%Bn}ZVzsIo_5N2?F0`X;1sz7p?LpoYo5(h$k;?CtGr(LDBt2qiYv!Kw znwS0bXE(p9H>qH$UJNzpT zdi=YWfdNXxcoxyGb-cN)g6jBr@m)<{WX4eMO-r|}w*XH{6_!Dz!u5f~#VQaR!VFUu zef6HU7Zn6VL@9^fR1kK3jr~?Ab;_AwCIos~G__u)*`I4R3lfZx9(|9A&#nqwa38Qo z8gG&e(Kj#j90JtF8#y$Y7|uuGgN|riJSbk;T>+qF$x#k2X=8yrUb$O7uT*d$<9nNhW;AR=eZ2Jq0Z{_0Gh|a)!>HC!|_@^Zl0Rwk8eOaRKIYm zzxy|!vdVJ@pLT}=3lWixExgd~ini=vK7uxm^=Q6kmiehIPY;=hoxG{L|I5hdox z@^>hhpDt-?7zU}jn25%Hk>Hw{&Hs2Nem*sUtO@|4g`J~7>p1uak&W#!19be->)eWO ztFgChdVU41@LL$U>cgaUoBK|7EpUmg3vBh0IjL#oc1rT+?nW7Kg99VYZM>a9IkR2U z(9F5z(+q!Z#nz7MX;*D&foHrrrPiq6^9kG~5_iq}K$)^cBtU^0X!ag+i4$voEXpK(@23 zG3F-I!}qO&K<2jPG0P?5y)BWLu`;XfP)-t#1CZT4hnHfC98G0NP7u#O>kW+fxgqtJ zPBtLmKz${+(+;=M*Hg%1>*~sUocw_akY*%`5?(b!cNqb9Ylum9U2Ju2>JMNU8f70% zlw1`DDh8e*K>&G`@o_hCFWY#O=0f;kk8w0gO5)dC)bRh~>not5OxwSASy&NXM5P2o zQA%0`q{B5(T0puK1nDlRoplYmL0Y8*6r?*;LXePZd`uWpIbyk8RG~3hN>oZUCx!pGzVz7o^@!vixf>DOW3$t!h(!r)x_Q{ z_2|q^K9TmjXm0gvll=3P6sYVhHO%QrV!S`p{|>v}SynhnS3Ae{Po(w5A9u;wurC5R z!1K#^a^m-omKfux9729Z`7ROlA?rtV&o zmv#@1HZoMPd&Z*mJE_aJ?@lLa8|J=zZc=ISOz>QVWEKYPDwYX})*)eueqAHFLzDTl z-xsgMqdy0tOA;1xM0kfqd_VAGQD>J-fcP|gLbqfA0x$>@0Nh56@&6oBt$`^PaHVT6TH#F@Zo|v-#eqMm93M2 zH9YA@xlLqygpL$3ixOc30D=6Ju6SYDU4cg~OgHo{wjRb%jrV$mH~}9nEZNZ;xiC$u zgdXhz3Ygn<5YyYlJs^+|i{#cDsc>99u5bXl7@naJRi2uexX^H%HFtgc+k!4!EnMu;-# z`n1PIYIe4&(@W#A9AX*zo5M9c5Y?*tAHr)t%~VHSnPQVUy5I6bIflc=;ohQW_4YK4J5=`Hhk zd(x-<+|!Ds#hJ|4vLEy>*Kx|im6QK9s9$Rr?U}NXpkY$&yxwRZX+5u`81YlD?fsC- z20a0(y_c{WSP>wcS#hD0SV|m_P^;pomhc zjmq8l#IVWgYq50)N=xMt><8otS`D9{8IL$shMkpyj?H7==puXuq?kutzt6@#i;Q)T z%@#rO=4Qma%=~iO*Dtb;*NtcgdD~3iNE!~YIN^BonYTo9^03IgoSu;Ar>ZBj zqIBh}+)oVq&xmtcdm@v~qvZEZUU}!U$O?vMn6i&&rD&TPEeEw#Gr#?Dc)FWEpzOKH z(dJpVhkTu1!qh1`v$B{jaauN-tPJhcU3dU``le@`TLL!~4NRZjx_)zyP(SZT@eCO9 z{aNoUrtbiwwlXSz@~P%2;ovvWGz-@vNW{zEUL$NFCe9_CVehAEu#n@enqNuqsh2Uf zVe+>4WSN0R|CO5SspZy0j;=XIymCk663$;dlX$xVK%-TzYat{%uZr?#Y_(=i8hhaI zpb4ybfA>?I#oidYUPGL(*OAY#`8fa&^)54IC0Nt1O1RvH_s<$;rK5oW;X$vUgK7s8 zpRb$CmG}Ci<%hb49+?`}xisd1w7yuGM-gnkpKR+1r!R4SWWRF?wBLkRk#qE%GDejI zFryy2>7!RI7n4~N2&!IE8(lWz!eJAP(Gu{604sZN1vkKm)*NK_C#~dm+mH~kvRmn8 z5k_c6O?KZj)qlWwu}H12`B~=2COQkHE81G+g$unbLJuo~@$!u&oU2XmEN{Ita-AW@ zGa^$9-4M)jb6f9{LyPX5N&~gr*A={4Z3d}eH|N6#FX$JeI`jcYP-G9u&;w$&=lUHx zYEQ)dXkVRp<#i5FPQoj^ma13!5?GVt1$E1%iXHSN?ecg-ylaI7sZN}V^Kv029ToYs zn&`M4ct;6uTvBBv+tI<1qW`xYo|PYbq5mOp@w!2SK$lCO=QGT2u~;d90_ECwEr7xv z=@a{D=V4!c;-zShmFt!pbu7y@3^kHA&t3o)(7CLN>d^+YU(C3qzMp$ko^2HE2k(u@ zSj$;Qx)VO0DXTi6a|_av+*CMG5jRm)y* zWi%Xy&zXu{edzy#;^Dj&uoH6`tNYmg&En{xQR|sUbP;m*l4;-bXqTHD@)-E~;>4*AyNgll z-z)7PAo*-IFns_g^0}b@{W=EsF;}7GQ%aDk^fCA~Dtf(2^7) zDdgfrm0{sY9LCqQCEIlT*|otux1{0v_wzBy4hyS(kK`LU72-Civ$Bh)5MjiyOPj)= z-au#l$S}?0w}Wr^p!cNl5zeDAXCnrmEc-X!aNMJA5IlIyJWN{d%IiLr{ba~$yK{re z3ds#AGJft?K5aSR?4HBdo_H>`@R7|z9+RzbvpNi!q)bc{xRrxKDLo-2*6JDjutEzK zeBZv(sP!fqGwYLk%xU=z%iXgNANoP_=h)(1>}3g<+miT2`{&;hB>1y}A}7mNGB?qv zVwbxd-j0rK74=~-F{JdI_g)e&D~q(`jxOSnKI5HR4G^QWo@IfxG?Ia{Gx8+z)%kLM zNy0ap6Q!R27?(8Gh}gNIgj<8&1GE0?{zsB8;VBDLc^1Mz3%Nv^K>*xx1_X zc1bEyE|Eey{$V-RPj_Qwt6=JH$372t)&azf zjm7~rDWP`?@;_s!(H&*K$7JU+Dw=u7(89U`$IIhHjx73yI$EcJxBpxU&$8(5Jws-? z>E8C+OcXsM*CgU=d)W$I%BvF#ZwgMaqNak3@6~=rsts_88=)1tGh)X!<7oyW8ntvaP}`j&;s~SJ;AJk7)tTy$N9jLj@OU40eO*#>$8F`m1VKDdCsMz^~On1 zLYZ!EF7MKqHdhHn`gv$-{$w7ry?Lie^s{h!VZqxkx@tqxnzTdrYTNb>r&ZiU_7 zD6JNn4mET{hp>eU!|~**popwZxoxpUFF{JrMra68L`_>C%2s~xuiv#{bAFFhkbMYW zYqp#ew_QwCmMmt7@Ke$apG|t9{0HUvf}W*&Q;+L4!oMwTKFmGxZi||%v&-B>y;_PH zX*U{CdH!SU++5k>)ueea^(i)o4<_6-foTdk-&A5N0ho%|g4T|KV!29gk0FU(xBKU! zZa$5K20rN0L zwG<=bjXHbNyXu-Bvg6WMI}mL{A{}LV>d9|e3|0jm97{2$KFL+EKK=1^n4wu%B=Dab z|AKW)dR*i*s-k3*rO1t#Ncs%C?WL87l{)>d2p?!e$;i0sj2&%aOlTH3tt?UYWH^H_ zL<|a%K?`YdB1b(oM&oZ433eg`;w9!VH-D-t8)?(x@XRkb&nrRvRIARIZv4bZknmE} zEWbzLUER!h%`0-5y1!?EVUjQ>0Ozc-*i!gb5B*psRK}e31P`#T#B(+fpVaqeWY~;y zRPA+z>hk|8xi^NlR=j)sY9`LiiIuC#cR-J+AS#XP+tVq2M-_{Q&goi_6#+?3W7K7> zri^b*=k$CX8kh2lmyO2zMzc9%;=XZf^wCU(@s6^b!9)@ghWJTI$;0xNf|P&gD@8n2 z=aFNsW4wX$Y#^7!SLpjBPai>G7k=7%2?qk^;QZL_`VX7vR*!P-t@7drcdg7W7S}$F z{tcx?yixxp?PVz|UHOAu-)E#-(`J-HAe6UE%=K|nhS4%1HxV369KizKOc+l8`>z&h+}=P+w)2s8 zW3Ie~)+6GscDkj@=S#L?dF|&L+Gu1Qh z@q(QQ7Szv4yjajRo2QASWLGPh>Tuwz@@}$Z*BQEq5C9cP_yY__oa^X)878Htt?mHx z`HPX&qtGl~WBMPx5W-#L{ytqD{Q^)V5FXYbZtM6`#a((=7XHr_6jf9lv}t-mVn#84 z0gvF%>ny{xl8>pJaz9DoIi5Ptl9-NAFt`mLC!=>LOp=E&UW z?7DKVt-z*-e6vq*vTBUhfy1mK>4dqpI-GREBk~hBm`7aC8-H|OkX)*EnG=X8S(v(d zhK>=&iiGfW%H6Kz@Za9zO1! z^6Siuh!}nJt}w41SP*+KFwvGAPtCH(bew+2N@0moMJs-)2TMikrDay!zr9MR_y`W( z2C9F>|8yVi+*&tUpjTX_cIGw0lIjFS8Z<*Sx+P}qbGPSjwpz2LL^b6~vLp>nJ$XDI zp%t3lT?*Lgp|!S|eT^1M4K}xKAkirOBfNEa|Elec$xQO=OZ5SelxzKs-5#R zJBnsWm@bqEqV(-*(o z6VnP~NEN0z@v=A% z@EAVZSz!3HuPJf-e^d@}uZxp_`#7=W5S3V-xY+s)U7yhEZvyan2(VPmHILJVUAk@c zWR9y8LbRgtw6EEPgfh(9Y58-fqd%oIM#NWG#M*|~bhm={`lV?u*<}T}b$m*e@8kKW z{15s2a##cs>As;(eqRxGAR0RQ>J@|@Uh^85&-^@d3zyYfOOdUOb~5ys|Ew=*e;--t z2Qp()byuWg1j!kJz6_=CcfaRy#Y}fV2kmACz}|=aj?`2<8*8&sS%kEaz=eQW$9Qg| zvUWEAiZKe#O#6G%&Xn1h91Xd~Aj5L37q{pG<}ZH&bb|J&cLyvlB&VR*axx$qcb;UG z>f7lEFVa32q5h4)Qnbf*Po~okgQ9`Um$oSa&!Ak1N?etU%ryG7nnx@qXT8>nio9m; zpf?~tkpYT}99iAW@ zbAAbk!XoEmu}v*PVo)W~K)w1{K`2Bb!F zOG+=K_kPw%8GQo@DC9ewR#t9od@Voa%UhlSGe0NGe@nzb(qW(C9D+BGN2y(h{Y!cX zAo{=7u6!>|9_BpVMeDIL8V~i`E4Lz<2jPw@{=%mT$~+`))0Oo|>p`QbMZcj?qhMh4 z_mjUvj`oUT9*Y$Jhkj?PS7uY{uLSUtVknZ^FM>Ep6yl!}6+B+Px%%OPP(=TLDy+Oy z&3$vACzd{d68v1_lz$QF`cK&cyklaNNqL?Tz59ahk(#DWaH%>sljAKbi~Ezytne5E z{z9qh<}em%)UBr#f)`mJ7Ev|~({(6qmqX(&o(nKz2FpJd25lyT@l>@xpE8%V7r{KC zA{zhH9JQ~b>yi~%`-X`PjcBOlQBE!$$VeW2l_TN2{iVF6S!3TufrP7rR@+V+GclrQ zrPKXn9vh)7Nn;6&ymuh*{vFkxw6;=4#ldNs&|xwCv@z^)R=RQK2Jy*nblz8L)LZ9$?7s19f7*OIPQ$|;NIn2|oiD|5@eQB!g3(ZdlmcGtWZ%ve zwJ?GUEc@jsF^)_a$myMRUg*jY+Zt($_UPmXPW$ow>e&2IDE@t+T0v29d5^eHWzq4o zC^)`kP+;_dce^iD}$an%||^x ztyqR(Hyr+vczzM@-UiEd9sd2>xGpL9_bvaT;A*CQ2Vk>LXQ>B0b8=Gl@iGC}|C3eG z(*D?K^-;{dE_+Z}T4m{PI84b0>iUyLtOB|#nx zzBH1VjH)$=h6ugRJnLhqE>r>f%WSoI`Pw*bn8I-^&5l(5YF8%^>kdkhv~i}^H-Ec> z%D-CP&B?N4*)Q1oxg*C1>ccTGt#PZH!I@r}sHo3_Z_zIE}ajP|_k=9u~Ds_of7CWw%U2SA&%DiAlWUaf!ejShfqNiQX1 zGCnwDJ9@4dTC846 zDHl0h&#VyX{yp`YoOpqgy|kHHw@A-?EQ6H_s%Pm)?BaTvx(=#u{ZL);^Djha-dfzW zEwM5;J!2Efh};DvjNiq7Ku-bGE6M17uw~?6w~>isru#iOM}komeCDLH;IGI;+W0l{ zj&D?%*9=2^Deu+;2n|HcZ+QU!Jwwx}C25$WH~pjP5(`=HDUnNB2LGgvmkFFEnraO@%qM-WZ$}0Pn@Aycl9+5| zgJBbWs5a_Bf^%Iu^zD6Vzl|%J$klaGewP>MN{{=7qefser(}~%<&!xmtsu$`d)lGIW#1~g9Qw0KQAc23jCZL8atuDPqls@>K#7~U2vW&KjlI2(sTWWYG0 zb6c8LO}W}!1$FtyLG}QA^mO5z@8cHIqP1c1ng>h~U5dw)B0-28~ZX{Rt3Gf)> z`Oscrw}HT~D+@hxl~H0jEdA)NE9gSWwW0cNBhl%OFi$x_S>P@`UV>M`8WE zw9s8yTS(rSYdPzyTZsSaj$-1aSHd#2DR&66itVqsY4_%Q35lLoQ5XRNJ}a&W+(J9` zBp)%Uju4LE!QrbNEz%|fiL{1q=KFdZD7h3&8Vi@dTa`?K8~b=Wo5h{oxN&v*gIjY{ z$yC`xs4&9Bak{7m%~-#lEyR^wGRkdy6z62C3l}OBWcAOjjREgb#z<@tlXo`HjBnvh zXdzt?^@KTaZRT8Lu{4kij|aT6vzN$0sdEkXiO>eyv-&C(Z;IT&o2DZeBQ^C^=CbOB~Ku z!$;-77;G!_(&e_Pg-r+{K+bE~UtSk%HHMNcZJu51f(Xk|r@V@`A9mCI{43MRZrXpK z6zQZ|rxB;*d)*stblJ$9J?rV%^8-1EdO*D=I(hnpzxYZ^vlCQL7yo1)PAI#I_X~$@ zXR5siXR00AcK*`Sol^@r1zO4O_3Zr(IorlW)1H?v%F-#%bOl4|@-T;`sZRrYq<&M% zc$|s7`NeC=@r7U2Ro8+l3JFe$i5)Lx2p$(}rlz1OSg6pGk&|6t$RBE1?a{mEq7RM> zClnW=%SQ~5z1II6c7>Q0_sG>1_-;FL#JXy2>Arbt#=ZMX%nDAZnW^rk?tpAtP1Ucyzj4Uug--|&KTNB*{RcQQ zq^D^8sNy_kQ`9Ir`-7;dIy22ZA4mIiD6BxYf~KCqUA8Tg<=T~v{5786A!H}JcN}AY zaqKD}_5%I~VSgT$wnnTYC-~(5Y%ezR@c4tr-5FalwFacF{z7F}*>Xuqphc0znp{%P zrRBtO=Aj2hvQFUEg_O58i+uA2uV2tM+1HFSp_8=kViX~nJ8 z6UjSnSDAbBVWYFQtTC~w2)+%g!rEN=1#0Ec66+;fKgguLEKC2PoBR+OYL=pDeB+In z#~cH*+1cI-8-H9aG*wZxBlRAK3Q@!NBH0*pV#14VNXUUjJ^Cy*)=yrnlWMMbYUH&? zle|79$c!2VlH9@-V8R5X>Aaim>?vFc4mgf#aY+|9X#u<$ui|EQvBN=P4}3)Vj?81e$%W|A@A zcrJ3jRk2FRLl2juz3inUgHi{M_az`-+X*`wT`)EyLrS2S3Cd&3H ze=SG2(0`4mv7}aO{}qD1(?hN8N2vpb)I;GtS+5yBJvZlz@vz4Hr+t?h`A>*i*$E^i zN!!|=#3$rX8EnZlZ&H767d;-}bk`40GB~a)F-b}iwv|gIW-zOJz7jKj+z+)G4V6hI zIXrwX7Asdh5m*}e1WCBVz>(8vI5sxH&NWG(@MS)-Dr6=?pDVU)W;>(sz-UQcw2gm3 zt-t*!Qpbbj=dnab3+_ISNjUNr9#w|1TlF|$%atzGh0ku@hD10^P8*~B&*%2tXYjc;&P(^BouML9p6 zr;7Vw6~OcIhahtiYo;E(Sk^$n7Vv;kqcmnw6q94hUjgom$N;ZEZ$R- zLnYRFh?@%02GcIG7>1Q1EfKu@A-2S(6gQyw_zv%0RH?Ef%i`)g>F|5pR>wND2LHL& zVBMS=hNLSZYtCY}WjyV4e8AfeyFk(3Jp=2lVL=`43X9!a3pftZKP&6M=W$IkEbw?B z$vmukY9h{=Lz;OUAX&zG3liD6oZCnIOGY_Q_s zs538>&!SKG1Tg*G^|hr*3yb_-!roXJ-H1lnORF*?sQ3>T&I?h;a3<-EX1@<@m|Po~ zQ}%FMue~u#;D~YWNbnuzPCqR#jb}Ns?BAicTxFR;ZG?WTazMermRg|m-45M-0t}Fz zP!cul7cBTO`;Hc@vahD_aMBy?%%tp;$;{3GiAaJ=8TVrk&m|eGU3Syfj=Vyl=|x~6 zO=EKsCOoBRSRva*p19pk2R8X_PVSlXqUQ`!!C@d$AdjM`(&Xz#G-v08FKU&_ekutz zjK26#11D9Fxvl2`5E>DwJKWr-tL+G<=jgnbJvQ6whqD$m^d*;r3M`b2a`-!v86<8U zef6S3i7dgQKQH0&txlm&cn|f1k(89W+>7&>bKU%qvpxTX%j2bRkRca|;HYCR(ko#| z_Dwpeyli_m7BOz|ZOTz!uJFjIu(RyjONsbV)SRgvJ?WmOd@MHnd@47g(PV!l*)t#kyC)XeR0$pM+2gyVWZX zkXQ84~c~YN(9uAkqc3w=7F7ClAelgd%i4!zUOHG=v<<^bD>i|F$ z0(nkjbI_Qe@Q8hw`Gwq=3Y|-SsI_(pv_4x%^$x2{JyKqM?)#^gZ&S)gQ$rs$Xb*W) zeYh(-Iu08RT@j+#s0ZX zfeQM+e4vv=R|`J^sP+6Wyw=uM>#6>Ch^iU-7%|QN8M5e~iL0`H%E8Yc9#OoKcK3Yt z^-lMrH&NJo`qZ9RG;M75YxPQGR{YSu4hX`F_7`|qM_jmdXt+zCi%!Zm)S7VuFb33J zrOJ%FG1sJGWKoQuN z3Vy*#amI4xp}7f9$SYQn>J)M0rJL2>_CwHjBy#cFxy}7rhM#b2Qi5bz4}AR!IjZ`? z?T=O#{Ul`?VTo=uQ~HMU4fjkn^+9_@R3|4rHmHw!2>Esg2gj=QVA_v#2<_jVN45N@ zXt^2<$uO!s38@T#1Z;UvcrZl*+vZ>&%z-XT)X-rWnQN{Sh(jqIOdIFHSQxP}=}UJj zatld4L-#H*Sf0vH{Pv9^QmS#(F+PW4%^F$xCZ>f&DHA>K`OOjKFp^p61jsNK@qgssk;vZHYi+|4y4I0)8o^^xoV8c;T9or^m- zc&0*8L)XT08)OPoC##VrVTnHp5JUCJKgLsD>@)59gPM!a-W6VHhuyLaMmWy;jv-Zh z4Mw1fsAzxg;)ZXLEy5O}$GHO#WfJqBp+^HMEGBq}Fz2s;O|^CSK=sc^V6;hS;d!Kcg3= zBW3Qmq<$AmQqhxa3DJtYKZ>DS7s1+Nmf=>?<*MZJE-E*M;Ow}Nheohp<@r;^g9hy5 zzi{?nxcU{Y_;O7A>YIx*EW>ki!ZEyK9Nu%q2FAQB{6WINeiF}1@ZQumwM)0>G>XKQ z&5yMGkWp&LZ}~%{=rvjP_ba=k{;-RZ`!8TrqjiT{%vqpxV9WD0b}jx9V}Ey*#Q7+)pxXA$F` z>1tC5m${zi-8lC6n+%PmP~=qkg0s>5YD@c{B#}+3N1Mj~dS03$mR|Ul!NS>Rn{(shM4@rDbJ63bgL;lo8QEEEA3cLxbcEhLiNTK%=#_^aqBIa- z&_Em3WM@$p6Tz*+d-EX*VyI6Yxi#~7kHI%oFT3^ziO@rceOQ7QEe6B|TK`pw^x}z` zcltyZYZz_gt%2L@iLlt;6}@l;^$oS^KaJha4-$U(rv!wKTW%)*fktcufXN)AYe+my zg3JjaACy=kpqZQ+3e_wg8YcZ=C{9fczlAy0bvge=-mk8!%{y3#;Z00;{jn zUSmGBBh9-ql7+OO)7l?DfOlXBh=82I+pAG-=-kpj&(FI~qOmER3$K4ybdim0ku z?0fMFL4pTQSTL=f={>E~;vO@pYVBhkOygH=@pc#}zy#I6u7QNKowg<%U#y7`zYqXV zwN2jN?3ejE5Oh8IiJ$M)`7NSH;$=NPO1e(O2Qbaxn@pGnj|6Mf_8`Ds*k`yADq*bH z2pvbmZ#ptO!*xCfrgq}*d2#^J*f`6C=68&ap1aqGG<2EZKX@W7EASAVn{ap>HL{&% z{C?E}kkni|-Wy?m?e3kMKyz7g*xCVnCPk*t{1;((@DT(0N;sEqzE&<9&D4}4MV#Iq z-5`pyHZ_9u8H;`$-Lx%}E%Y0+9u;sf6qywg;N>AnZu@uGsMFT_CG~hAGXj^h@4e_1 z*dZ3m4a(n-855v^LUIiQVuCc=<~7clB@G$!ikj#4h}OieuqIqjG1X&aVdl{v?242I z_$W`rMf)A{C#lgs^f)L^^`Z4!%aZD2i7qY;0u$-}-}cV@-vxNF4EhBPebK7^?^l?A zRB;mjl+dN)+Y`ZuhVo&;p-45i@hed>!r{pITbM9{EmtIUmIepVz&1U6&piog)Wj80(|by z6OVP6O+U>`Dmrn>F}t_}Vzyi|Xg=_H62vO7e45VBg_NYGtUt4xAhx>3jI z70ghvFFc*-jmwXj9j0nD4lTu-FWapTW)I-zMiEpucO~}yBQ zXHxyjTk;lymx4o+qN6EPg)kHW$5gn8Z)BvTvV{QvilLw(U}Go}@i)&!&|)T)f)5N| zCS4dcUQpy|6bAPuCBfz(hkZo0PeoHG?|kEb{3me_Kh~tq*gVxAniOBvGpp=ni9*yN zdGZD##tVL=1((a}bHWf*faFL^DDBhm_{TOoaX3Q}ouct2&0*NQ_c-eyo-AZwtl|AO91c;4a#mq#E=Y=lNl`n%V zr=RXW0j@n!NZ~FfH0m``Z_e#2aw?_}IEG@x=*p(P^;<`gBR~AAP-P>5t|D#xr}S%= zPkw=|(n@Z&!uv@G^io4wcplXi?vIGiy{<@sUJDr2sh)FdDtaZ~Z{LqeDZ*y}pldC$ z!MZlG9qHwW{6G%9R20Xb$y|3^`k!ni8s#YvBvZ=D*LyPBxP;nY|BvBYk@%B0o&fBk zUE5{yePWB;@EFXT8Ko;W;xsuTx>Q7Vu>Zpcv^?oj6{&9Cv~kng-k!1mIaExaHiVz* zG`EgHsQ9r`Ir>JVxgSddpS-K<(8BOTdET$lQjylYeH@X6L$DzvMIe&|SFAC_4bcXh z-F1_5|2_ipYlEA*bDpML(@z(QBE5pLM0NxlsJ|jQg-{B z!4IZj!S(CR@%+v|!o|c^?I#K&#^W{~Sk-%UvT{!derWZvl{?F_M{=yA+(-l(D2H8B z*mpG&l=#=dwYIbnr=BZV-Qf6_{rZlGKd%6{g#;N$EU=%04Lk`tvZz=CHfi~bFPQvm z?1I{qTX=&v1|ZsRl40~8@!*LQU+h&nogAbSfgXuUWWDrZSY9+JBe>~F?8XCwyAQ@( z5;LTm!pj`UB_(*|tW7wTaO@bO&rlo4a#vm1dl+{R>!8ZF^k57MVFazLq+GA!?!C1W zfa^)uCk-x1TmuC$Ii>*o&P9&%yGi)5bB}=+?zX9iVSUf;y{uBaYO*xBcz_d>B?qr8 z7d5{XYH`?A7nBSNMQE3s-X1g>J}*pxI0{_Qg)$nHgEP>+bIqvk!z-%c-CaEh044TA zKYCVlkRGXeQI-jdoJn^61g+h^i}u#T?dc^+EC|Y+!F9Nm{Zl`?=QV;w5Vk6(1`d>X z>SYIQ{eLvgTs8XX>709;4W16ksyWhZ`Ppaq+Cj@IK`#yg=)98jfx%2Z@&fY#2@=Qr zy=HPf3p~h*AVV{SG1C}m-E(Pht-lZV|13ZZZgnEf^Xboz{PzQg3MkCG1ls!wFJJuY(lZ{T`hL|xZ0QI z6Wr<@Dv5n&;8mgSf{?R1Y~DL?8u|cB_}10*jOo1t?Xm!hjR>-PX+suyd-H5)MYW!P~qM_|pl-+m#-y$0X=t0hxiR41aM>)elD++_^tc!@7gJL)WjR3^W`mK!Bm zbDaJ3yTH^6cq78GQEWrsPqgdc&9tGniFGs}PMeiXD(zMmp7fl2uq+7~_P9d=dQK0lupB|~JU)*ez@ zK(Nb37Txi_lZkHKXl{TJD$$$-8k*>wp!`g~!*I7+SHD7G!y6g;4|ue}+YXot0o*r= zgJ`<5J+$g=6#@{Kiuejbg(TRwwM&!)(0AAv1{f-~$qkY~IX8mypQt>CDi`;S_fDHj z6wNIP{5`CIO{IqNR*4Ew>Ku~Z>5?|k8oWn^nQyvO=S2MoK;#}FB5|T~9_z~X?(QP< zpsvbAmGfVB*$MMcp10s7P6%|ExsKUI8_jiftBJP;D4EoEHP6K4#LWy(-6hg0cr6%~ z&qvAQtiG$BBB<_5Mb2al^hT_TAf^g93*Iv1wf!&N(g2w)b1-GG!6u^eRUl$I5V26U zOda&@_UhwYbh*3L6|JJY+_Px45D68479I!-RjAq#+gIY}dX9o0RMPo2$2d<(L=tv) z_jKfri%IS+?_Q;JdKZ}|Q2uFPd_ntO*D+*;1)xAv*oy540*+`60p0?u+?#vYWl;cH z1stGI+gLxIp1Tmf`8aNWtX`=K&+xnVh9o;?ND!}(T&+d%ba+NbK=7xb9keJyav43z zd{#jM_CmYEM`6s|@ZgyFEQznszHE6kAFB6o@7qLnn&1xTJlYj%dsGx~e(AqMuoF)P;Q#CJjj6FOaQBgd$Z*Q= zkQ4&4`IEsGjyZjxU>~zx6Kpa!O^t&Vf+&xCR`Dsb)6P7XvzbTRQC;j|pkK1_zK(ah zJ}+aOP+4P;MW>{sMN@6Bbz0#z3UXA}Z6B(vrM~R(`3+!&VIt}pHU3`wgJntRc@Q^1 zi4BD#pcfGV<$m)So_n^xphVE!c6=4Rz44g<>C(jTQN*!MkkR!1SMNA@2*?n71N!nj z3ut)P0q8dh4tw9h;PY}ggg4MA2L{ijYbghiQvJDw5-8;JH)1^fV58vNRYFPQ_) zWoc0E<6owR5XG0SmT@7O`P$owmjwL*GU2Id3$tkN$2og^B!obgqtI{LAjEn`!k&7Zu>|d5!0iF%To8KXSmpO&o_Ihg!h7+^Q0)w+5wkK#(%{=d8PzA484H7ua%{ue_rK(KeTWTIwaY3 zNZS4PE8vHxh|rF*M*Hz-W=(}(s>#qPztq1@7By58l)^Y8yOl9*CKsFLCt<3uz0ncX z&P;OGkpH%1FW2r1y&-_cOnZW9FmiZj_;p*PAqnZ^z!#$Y9K+>ZKS>R zS&63O1kz`{YrqmmG0_@3*5jUrO9n)t=Fzj_I=1l!%7z03jT zUUDhHMyl&<{AJ5jUBkqqZ6;(?R+)ov(P4hr>#L-r_sriQJ0(ID(~>;a$zZ zzo}Izz50tO5PDP4*;y(V(AY49@UYYs2AleDYfPg|=hXFbpmhCsV&Vl;3o9 z?rDl(wzIQ^1FAzSi_v|n`^zoeWNIUaGfmiELthH7*`i4sxIcaaVthmpX!Zds9~F2~ zF_p(HUPHaQu~Y=8=VNIqRkRse)Q;1qeYjnV#mn5S&Lt%+L}a|CJjLOLKy%}DGDP2Rk zWQg6Lgi*;EZH_EQ5zmcPJn&UpV&<{F?<6 zi!AJ+$62s>nexo{qED(ELz5fYMX%q$_UWv9JpRz3GtBNE=b0fUcC_XY^L+SgpD~W_ z$UsB;R7Z770xL3IjP)&DWS~P!BM*Yuz^jHKI<)Y?UF=~+s0?uFc2IIlF^>osQg!CM zWa^I=ai6RU#fZ^lEULhKPU6S^WEck$|J#Z-Qj9}iV00&?=MJ1?T9NvD(zX+qAQrS|DriiqZ+ zDt{fjQ1>PV-O8;=k(hFGuP(Rq${&;9ip|<(+pxcO_pYd+Hp zet4*V%tN&zK^V5I#Kq=a=! z`haVw7FkyA^K{A6iJHy*Co+oe*VgOXmQQ?*+XGhBV0z7E@j@T|uB&5&m}w)-`@i zfeVPHRMEhu#V)-e9JC1Wp^z!S{@XL}ih8@~gj_>-F(D3s)~*9*03% zCr0Cz)g>6tym0)(!;#WWOiISeL@RPY;j-FDG|&ul5u9AS@|YUPIM$J zgfzK|U*d18x;LI%wX~Qh=>oI17YCU3P1aB!1Njd8>+qh-_#b(Yi9=9j37=b#Aa++J zJQu*@?)!Wy{PhKD#y1p}NI*l+$`>eN?;|JEAk!%LS(x(BTJ@0LoKuLHc$;{bOqcT{ z$7yN2Pdzk-Is=IvE5t#P#zb;TEg{1=It=BC*h@#Zd;6Dn18>lKBQ`{h?*exEZ;%5o zX*imIC2_z6<|H}UH4RV6lUf5S`o&WFf2hTe0cL#}Gfv<(lUD0GZp|nP+R3TAn=NAzHD<>`XdI+u1x0UW%ME-SsfVeaNl*WHdDlLE z5$es_@ejjFJpLN;y~d`g-b|b%w@eDugG8iGtLE`n*ZEbeg^H-K_(N6XWoe;%UI?=H z4NE%{fOb}T`;YXmf{4dJJ%d+je_=kI=UTo4Z*3@Yp znX!H5%*X;WFxN~kZYIA;5$g_?G;M+My+#GDA;BM`haE>pZpi&21dw=c2qZSSZmEy0 zpF&gz?*WXWc^bgeWteQKu=Anqn8{3pH0Q&ke6ZCLlE_C877r%!fqEa)BqkcnF`a2Y z**M@2REN24;6sjtpTFAufm|B`q-NdU=do?t?u`eKfWz4JD)=0h2BC*Z#x#CFk+D@d zbD%Df1#iqkeEayp78>|QCipU)L)h%~5mhn8r5pRJQ>nwL0@Ju1v9D)WYl49eIAEZg zjutRS__da%E>btG;skWa0|S^KRimC7riT3@dpB_of?c%G9;+h{k*Frn$#=U}&2w~r zZb!iYzUW$dS?@2t=$Dx-Oka6>C?_Kh`4EwgW36KsH+lJ{K9c2{V#HW zZzEe%Yqk0>HpDZC1`#JDXFrdIi(rY#?k%J0!IIXx9>$gkiCfU;mBNq*#B5;RhC~fSoM?J6kKwG>RoH z5-;%6d)la@RCCDLjB5A0KV48@4&J1x!%+l#Z^!~d@5F>{Q0X7yW%QMaJiJA`7pnfG zbQV~@v#PqYyC25Buz6(vq6c5-F|1z-QWdy^71OW~l570Lu>-xh_g(Y>?7#|zY{>=( zC=1_y134Aa>Gbo=^jGlgZ@$hb?X`>kU#QG0+!H+i4&qF-ef+oo{{4R+3ahJapqWt=5WKXJ24F2J<|5sLP-g|9ke&Lr6R)gp+wG+!MrNqyT5Y zM{=kp_#{(P>v{%T9uQq;!v%V4oFHrm8gbuu4glQq0v%Bt_Uq8c;4RiVMR!E{1vd^o z#R)BevMYXYZF34Md)cA?jt1lORVFI$8U4Xv%nGd8xT`;4u$$DL1qKP3G7ar(=* zh*?cC8SC=QZaS1y-sEvOc~gCc_hwAaYI_8iaq>(~>qN3#v_tnqPO$s8pjv~ZoBYYf zm-sErYRsoAt<8_OncU3F)i{25cKx5tp}P};;s3~Z(OaR4V@4gTYnF%lZTpwq+?xoP z0#_EWZhCMA@4bK}6?(74{NMY-A@3bM`jPcY_!sP3u^U z#Hnz7c1Ly33Xsm3ZC7Ftz;V9sH3Pj*_K|>m+#A?A;JHVCOUSWyzs5#En`i!hBTKk( zHn6OOv*!WkQxYizT#a0n8}AiUNXU1EnOs-mMDmKL${C!8?Q+=S;UDde5otIK_got< zx6iuY$T8Z%IcleojaPawKURQB{$zw7>1-d3xK&5{%4a%F{Z|76o^2eu!#O&7T!%wM z9bK9G?LUQeUijuus=;mXCtUuByu=9>57F$q`v`&I3JsP)`8EwBQzV1?ljY5%2Aka& ztb5+}SzmfIzO!QU^BL6Skd275ozNI%nEv+Hl69sm#vUf8TLU?$B zQ@=UmSj!pmZxdcCkDf^O2qqTp923P%C}Qg>WSI7;!zaY-(($H~Q7JLy?t^9}>+PJm zR*j^+xw0!uG_1NgnuDJoNtyE}+es$R#inWTZ+5QaNHp5B=^(sS2)td_=kW#G{oj+8 zw?%(TG){8qUgqmUQL%g(3(UwB zgj@kmIq9adzX2Dc*J!DZe(WlUz;vST9rE+>-;0NH{{6DkOYnRYk68K>dZan`4?#=a z%0TIbz}UF;#>5jhwwMOr^*s#SdOLN+<miJtSKJt@mE}2HPS|g!>spGLuYn?JPGKad#5?M}3;R%tl7}0$0;%OA&3z@Fp$6a$6IwmX1j01mUTACVh(u>8)#D zFn-qmm%Ewq@og@l#wfAxidS<`a>lRKOmP@FDidM`A0 zZt+i1Da$%^I?uySw;!=z7r7lU?F-~maB&JV$_JL}B22|e{yjN|vyu;IdGX}20#I{} z54h|}b_6xx0%}~ovDzT9P#6+dL(}hV5*Cui>#P^r`xz5>jxP7xjh3)DpQQWIQ_e2t z{6Iuh-It$Bd8l|i=r_5nKN{0&~Seb>nykq3cfFK)McFlv-cbyk%00VxgS zC^H#K9E4Nd%8;QeV@iWkqD-M^IE}w#iWC{Uk(8mR%)?ctij>MwBtue&B3$HM+tfbn zecaPO@B4l}+h6;0&hZTES>t!DXYHk^lXVdCZS(&AeU8`1eR)?#^3$q zxNXA4Sy-f(5Ib6i3Bp4o&FD|5wx3a9D2(tS)s!{sn6-k4gm$WgViD)-wSxO)h3eeW znBs7H?vFJ8B+-YN zy@AIwY>zmihA#BXNS2cU!O|0R=4>#=6s8HebTW0{U*_# z87v!4qN?eBcDk*-!N!sHLGohvL(bfIScZ1JK#*NOR+3GKSv_#;ySn+%I2}w^ z&8^-Mf2l(06~_qXQW%q?ZS`^KEHMh(Wz8L%*y1*|8cD*dH|K$KwQqL-bVx=mZCGhi z3xGM5HylEHf=|Ts-|{#8^J2x7_+g6aFQ5&;ff@wvm54k@J6&IP*y77tL_Q0{K>z(X zx@-OLVD!T`SWwBLgQ{E5NQx47PmIeUpUvs@4YziWD8P7L&P^#bk0tYXDRP^=9`Q=# z$!tM-yhEozS=}kYw}+ae!V%DozCw`f>wiJPgcY@;J)8@`DGYbBgWgew!?7&Xx<9r0 zSl80IcIOv^1+)jO`7M4)^L@I3J$yAmW;Z>|`y0h7W3!0;4?O!di(=4iz!GKJ{hQk; zr!y-pZp|CvctswHo$KdwUyq!Im?+=<=f#>`5ESXlA`!@{ElOBqvD5{!I4-|OMkzQC zG1=0$^G@WoQP$hUT|*WWVM!vf3m-^vNZ_Kl+pFp77$qDj=0(Qy)q%lne&pT8T=x}N z_q~5(;2zA_*DEC2FIc*S@H+!m*BWc5fLQk3lc*d0$>pE z;A`DHl$ei^-ghVP;7rAWmt$kYDV?LHsgyj=bh*DE@76FMUN!-H8^dj!-~_5~_L8s- zLw0!Ke9rtLJ4Y$Ha;1?~(sJ3^plfv03>2CIx!mwg&IOff75)$Qm_mBlglpP%H5Fdoy)8V3k85sUfGT#d8x7g#CC1JW^+2|O5Tf_<)z>IR<+~C}DB39M zeG%364sR5m4*T@w>>0q|t-FR!b8 z8@+e;?L-srU4}@HY)_z$WyYE-^nm^_`9bfnI*ijhrJ6LoEUS&475vfu%Vps*D*SYx zN6rV10E%;J)#ZCL*8$wZIobtN8?K%Ocp$;nbymY^4VO9(mp@vhCIfIV-NbSz>&Gh zCbSVoLe}ZDwl-Rzg6q*#)zHAIk%Abi|Fqd`OXN*o*{$bv{qfl5nc-CreUC+Oe7-6m zPVXK(JB1DlM#mSsg{XnWcZO>4N2XU6-Qf~aQW+cssnQuzGBQ__T?3N0)P9QUr5uf^ zdZ{#1m+V7&o!5x?a7(Tp1jgYkd;pp({;gF9YL#R1#j}Y@j@~iL5Pi2Hl(*;U*GjIp zE#76l8%#ifMsb>GouY&fyvZ~1D-d6wt+Kh{=Q@MXz&9bR4uZ0KyKk9y7A=KWhQ+de}wp5kZ~G2H-CRiH~%!Ppn}3 z%C8SpN*1lIfzzj!v5I`0Uw?J5^G}H93YK+y$=jMw@dEP+!^tzH4Kj%N&r9t9yF6a;^-_6Pi+!py}qXQBtE#l-jYX<=H& zLkC{Nd$gU5&vm+5f%hNsPk!; z)SJ|23Qsh$x2vC#Tj%lcGdA5+NGe0DUr*fX?ORDc^G#1?@-n-LbhyiWP zcic`~F!=4cZoZYDg+ap4fX-zu2QS)Abzb^5<(A54t|1N4MM*tJm5_CIT{L9=_f9iB&o@2e!Y5|`rN@<$Bp&PH;$XUh(_dMF_-IryYSBs z0s>y5J|*TZZaZV7(@62V!b^IbWQXQ@^oak<_pa77v0 zT%sZ$52V-i_a5*+;IKOFkY#aqre*{exB=6=+EbGwNxSRIgyYPS_s;Ayhgt2{EgPdd z`XOR2DKj+i6@7uE-Fk9$_lGYe3P8p5gtZa z?8tHFt4GqQ#6#uOcS@hjpUj)%Q#_BNn8}$C^5FC~M?){8>9o0XUsFO_e!6ylqAq+g zR)wxWu32yi^F#);1U1BW)f&Q%h~cA@;N>T}KqEcuH}C(03V*cd9i2s$vhcgYn>i~B zZq|Q2)}~RtGo+@$B}wgCi`YkUsRZzpQPHC0nQ#~ed&CU9ki+l*Q)IAU*7$}ALuW$s zo>3tfeWPCl?JMMbCheDPl6)WPnIQ5Zq@YdUf}Sd6QF5i6)nTawZnCd7?Md!m{pHam z1-uq%8P~f`42Evq?HXaVt~(`=nn~i_JHIZf%&}{G8#rh+B6gp0-9xlUjmfj#Hh0*{ zBbDtrtH6^ex9dR}K%nWUbw~wxu*mj{6sqGrZ3UZ@$Uh;2eNX;$?|prt=X!c_Ui6Yh zUTy~$iLGn47NWv!&4loT$Jql_F%;(`_30;7(+)-EtkC*$Kd0hDjKt!wwsF zGjgbv%EQg2eYYS&-(_QfS*8(7=6Plyw^Zd`FH7CAPmj`~oq>{YB z*tq;~`4ObYg(5AgqBOk9?kOE$wE@( z)^O?4aEWvTHV>r!YkT>)`oOF7WJLwvbOdN5gBwU%Wgw2C{OBs!I8(GD`ffK3xg4gdOE)w~KV%ro-ar4NV}AwVub--Cg2mn}C<1u&3jDovHSq z7Px|5T?Wl{{^p8A?x$qm6SUJs34=pwFl8y(v)=!gbpEnmEVMdAR#Gwz7mm&hOAww5 z(BFdWu7=c}P^;$!yZbi4%b{$1RF@418^C2*k|}2jC5bcJzTL+5{F%LRO%QO)Uwlcq z!?x+!^894I;`ndgSJ^l#JadwDOY*nuk$X6+)8?adH|#h<7KG&N{;0KR1{hn|-P@1= zxh=IFRig!$KBqy>rK6Wuro3cM8^1^R@#o&IgJJ)C`uzFHq0L6nS4*08{*^2B95UNI z-XHt1%fRK!XPqCeKPxgHrOMtdbe|EKend#X0&J%aw?=mO67?1QJ|cocS%xPPWKV-P zC3omjKO-kZTLCfS<(RkDsANop6G=}X_(JEq^h_DG-W>0qOMT~d-<9btaz-rkHL3Pj zFPeMeL`(8IE?E0-^zLbgY*~dghmOmt+HTg_d|^Lpc*D5Jr95yj-2qfx12IrvaLy3d zRLEUzU=Bxm^1TowU^-1)KW2 zvGJ_gq1JnTU7A7toXp)@xt?sE3WWX4&6yu_u?Q0d;uP})owm~~hVz%Ig+>Z8zCT;~ z{Wn@!H$j%Pe&bhx3r>G{Y}-0uwQIm5UPxb;mKW97^W@Ilj-W$fJ{MA>t}Ju!&-(DG zdHy0BC8rCb6({!He*9W8lzL!VettAKc-l5j@XjJ3z_+jOH5q!B$S>&~(WF&EwW4uV zthQ)ugw4{h_Svho{XU1b|8KCjD4-f=><+`Mf6 zelz=Be^)&;T%C9*#nv(CmOH6gSMPafpt!{c36W9}TC}(B##NQ#(Fq&xCh9cYgxdIZ zAWi(I6zQt|$0SQx3Idc5(*x;h4otZ;6f*x84F~no(dn+}5^d*anl2@&z&n!K_d!`G z3$^!Lis@;!l78Hi*ZVNOHY<2~9c1qwmh4Y-V231;@_e)G7Y3>NhT63fw93y8%R7Hu zS9!R->vl9qj@T72%i{o|vGuUo{0Z&nmzmMMiju0y3fb#?0rxMvTHRf{cl7T|JWw*H z=0WlMgWLUnO8Vz`P}3X>t-_Q962jtQGhV&f2-(Mg^vnMI6!~pW(&p(zgd%Q%pl_M3 zy(0*hE+|@O>tL)p6zFO@7mVPnILZ0pA9yVP3<|AJ)cw`bv)`mc+utkiolpv zW{|Hcp6A;93~o#|b9A&CBo*_!PQB6PywP2%cU8ff2t(4!*HJxtxs?&EZ$@(xy`5(% znIF~`|3=pm7>4b>DTq{Y?M-Y)bvlF&D=K)j(SZeAxN#RL2|m<}bUN54pvXs_zk ziflS@yr{`p%)N?Gl> zjVbcUCSPrBGu!IBXK&PVG2YrFZ<@*)rmW2W{bG@N3Lmh_`r3D$Ycq~IOCw3wtshqo zfd_)%V$U$#jfG87$?p-^t}UFdlz{9{YM!HXqP}!XYs=SWkFV*+#E!SO#dUm8d6@r@ z7PcwA@^3btZM6>^9Xq0~iq;Kgsx1ug{DW&rwW4{a^R0wOq(5fsxwuZdDGGhX?sbUM zLcTrNmd82+GJ6JrMiN*>PheM(%(q1JKXeBQ6+!zcO&!b19;__-`1p~VpIynet@X=_ zQg@f0xRO=_EgbpB#4EX)0+Isu&aO_p*Os}T+*YGhn%FUGQ-q(p#p=VG z$OWP_YWlQ%Kk_?^!PaDPqP{xd#%nVP+Aj9U^acRB??m4ggksj6AxmA1 zZ806recr&-j(lG9(bKFM8O~YU&Ao2Vw|QQowe@7ZJzqD~#U@Vm;QNYLMe zb&K;YGSn8nr^wGM``%kMtEqKcZ+qz0N8NpI4c)6Z(d1Kpyt>r9clLe&H5JIR3TpGR zdwzy4PI(jsEBWLJ=E3S+F&Z8ib317n;Bhs??S6y4c}3%9+qC6M9TKvZIE9Be6@)l7 zh0JN;_c)e5{qLu3O2R!nV)n~v^;eq?2!>?4+OE*mTONCOe|(J_f^|8(zRM^5BdESi zF5S27#Yb>I=aSA#KP!kSvVN!VuKYrCF#=wv!O4Bz<$$gZMNv%%q)d&jGM4PSpVnD$ z(?WGC6aak(tKPP-xlKu6Q?AmS)+L@V1PAvAH%~V+FJqN1mVQioRMzpjT%<0b!R_yT z8`Dzy^mnIt?9i@2x4PrZ-vV+-UMcdz92nvjmj&lj> zW(VkVDA88;?9{f|TU%_PgjoT=k0d!6V@&knWX`-PG`?^;OU5a7Ij1M8SL3-%mQ$)x zxr(#Mlu)5S|LIOO$1d_a&3N5%ipRgLs^eu)WmO43lp=iyxn?e=v8U^*z23KD|9Xc` z?RsCUMGEEw-_UxlODeoQW-mu%dM5ws75wVG@vX&6s$AhrDlVUtExB3oZuAQ8m>n5@ zxB!O*?1NCDQL+$@k#_nk$%_sD>xRJrf0A*Wa-2wSr{ zi3L+dmiki59USE4PX(g`btv-9?mBGobnL6Pg0g#6ABLWz`QtT znYTT>R=jnoic={_3CP8yi+h%tB-KFe)rJ$_GKZi6d0&J4>S(jZ{pKXE8Ih)+RF@s| znq7Y!d}wLe&8D8Y6KE<$B${EJ7^0!&X?> z?X|xCtnm&~1rjw^V}*l#OQNrP^P+?uq=gV+6zQ*&Lf}z4!5Y%uO7p>MwEr$eS8~c~ zwhT$HLwInY>B-FIc%|+(zW=$o2u}$O9k6%#*8j%R*e6~({b=lhfG)XNhclhqv~Jy* z-=?g}d-Y?>NrN}fPp6F$K5;n&7Og)`__1bZ{U!vt=;(u{kZMP*&1nlRC^RpeO)|8FTcCHCp-3Hi6 z@}aLjTmASCA5Ck$<@mtxO1OotbE@V#MdK2oGmcq*anuV14QF>oPF)F*{vy*yQrM*z zN>Ha;w8md*=#^t*9Q{pLc2zcyOlZ)ZP~(IQ(W1I{%I|pHHe|WfTyYQ0x^nLD^rPts zS?|6p3>85I#hj(c$%Ti!rW{W-bK}c4x$u4EMOzaZ+7;c0B-8Gk^}?u%2j%QA13-bK zuea%fb=qvGpSu%z-KW#MwL-39ZZdf7Aj|RBYbpZVx4FHmTKqP-^H1Ze{LV&6TfaOA z5+WZycvP^YSn6@nwZZOOMSwte76d=&%=m@hgKzis?Mj~T<4!J}eH@(Msx>e(w@44= zi7^DSOsFpM>~wFm*<*OJv#H>q)>`Eo%|T~e;zNwXzApG2GtGI!^rO3k*xPKre3f>0 zwSqcP|A4%}z^$><4!-2KEy-S(5u+;d{JE%LNa|T3_C}N%0}y2Yf+piL_FCmGfDw*H z52y#IB?e#LPhljfKKhG&`a5=t*TFs4R`Wx*+ANoa%3n6%Xc7_(?d;WeIM}LP(s?gC z`bQVAa)n>U6}8zy7s3M)`rOj~mRT6teY>E>sZ?hPWU4Op%SdQ)eJ{Eq|6Nn#=8u$= z%Nfp%M}&`Lef!s(`(col^Fz(4iUt9aKK_T);@xenf^`%gp}Zh1GPts*L?r3?Xq2J& zLDZY=rfwQ{`euT@=PX40ShSCB*b=O;K1~~3c;m!A-EV=ORROCt9hIBQpB9B&EEaqWgI4wNJbGf%yuh97G+%s=E4eZ-;?9_| z0PH&oSXA<`*SLE*qkAysh&&M#H`?xHl=`UiLA>Ntq+H@~pA;Q7Fs5EokvMtVQz@*w zv}0O`b_bBF_&4n+M0@Nzc&LJWaM|oV5E_`a*))K%;cZQ85o*=y%$2r3@(Mq6V*(vJ}BL<{^%vC6$IfvJ#(giKQrZbZOg8sGr zG(DT3$%PP93f8e#a}jo_AcX4>s%~409gLbj}Cy68H~cu!7JM|&mNV2g^0A>hA;^JN$kbP^4g2yE^jplgCt3o)!-NHaj6 z-;L42w6`oY&cY5slqMM!uwODCAq*$7eCMi>_FBKb82+gx6>2P}>fEJ|FUO6z9=UU5 zwh@Z&CLB2Oi7_ySdEAEy?Mmr>Jqya}mR{-UeB(9*yP!anAXY}xmA9;9E>O!zePbvqRf#an7<&ZP>W=9}^zP zE)o^dpaQm;P)Z zhC`=m0QT3zx{qNBmaOjpL-?Iwxs2)jjx24*dg=nJ^|^R{xXD&fM79E78QBK75Gyp`6>HTnvc zapu866f9Kw6;pU4J??DEm^cTl@T!&pcEu&vs^hgTd@3>fQ4p`ToFh+HXla{ z7DDV*v~Tyw6^Zd19bO#u2C%2sjI_~?qT|ms7HW=@(k>!*7sr1!_DKUmj`G$d%swCg z(QBBD9A1#Bv3d6NH6xO)Kv2!*A0~X*a=-({0MTTxEezwgoqXZg|BWP-K}PUHsv(A< zYJ;UUdvd@IM-uJ4z(B3y%fcrnVCw21CjDK{Jg{kwnS~uEmY0!$xg%E z8u(IL4Bc(xF~%HvfPT*4n`h&}%_1hM3`p2_DB&GiC4v-h5QxRD z!yG0>&NYy;1K(UxRqS;8ybqB|blf`;MsD$4dnA?)9RVj~y8fPgjau zLOXr(Ai7W$z1`)om;F<23u5!$Fq_FEH~1HN5^Y`cWf7>B_uzM?f?UNC`iOOYF5vlVkX87za< z2;dqcq18B!1kDDC$kWx0eu3@dq!qwC1eNrYHw(g^MxPBH+fO?~LzaggQfL5S>d2+JE*ez5I-^DJ>0r{LeQX=nA~YCK!I8eS~mF zzPxN7&Ps8RdekbweZ0fEq(*fCq$2cYcK*98{f^0MR7oTq?NT}0)B95Q`LD;JJ+gKl zP4cYz{wKS0DeFJSn2lTS$mSxY+ZRTT|&2VENe* z!8mXl+kJ$bvkGsHn;1QYSU6C#_h>x&t}p6*#eBdv961_?1EV4X(-Z)Qzt3JJj~*5^ zBa}OmlyJ)(hvuTnfsOO1G&aOR0U&9?>VTiL7#?FPoawCE)R7l;p%ySqzXlBeD>fD< zZo$`Nv=$IN%Vb3(oZy=E+H$}aZ8i@#;J`kV%etV~ot7Kp#1Y_BkQq1?Lnked+z<&2 zM=H=K*lPF$8Xd7Ze5+V>fPslBJOigHQndq5*+pR8GeI3YG7T4KWRKGItFTuA0V?86 zmJ~xFAI>kJ{ulu(?7s)#Km*Xfj>ac275_lsUIIjIl`L1Z@o2FIXfdP8WDgGGAZH|x zBsF?(Zoqb3pa~iR{_HM~WuVo8T)C9-*K$zXE=0hey<2zhWIzj3h?t7BL}T$%e3t8g zKNlani{0a5h{W(NGT_hqa#wJm3FLB*FYe`@VKHEMVHvhmDh_5+wtUck|9mK%W@Dw%ijd>)S2HfkR*d6kvzaPO*#1NmA3HCDSkufpL!WE#eh6Y?q)sP4mcorkoD z3AEm_QWgA)V@&=F?o4flooJOi;9!eT=?IdtNJJio-UxqDw}Te`R;Ba>PqBnTEZiKM zUCDqL$^cxK3A&cy_!7`-4~P+QK?_r}4X>>t5OOp5853U1IHqOHp%xM7M=qYs_^fQc{p55{t9O8$k&K2+(UyK4Z^StUjK`DE2Sxe zcm0{_fZ2OzghLti06;|A6v_IVZH&dq9V1gm#DKu+G|D4o)0j)~g;VKSFh<~s4?M^% zle=+-cq4p5?!<5G)xlHrFe_XL+=*NW#8oVi-&r{R98s$k3xV_aV0IfX9p;NDu1yaf z?Lz&}_~9Utp3f+eo^uq=^b7kDJRk=MLskS@rtS|oi^t)BCp9p@laK?r3m3bRmVUr4 zW`Q3-H4_!LnrZk9OC4xXDw?0L`l0CbqXQkrPw(kW)LtW>oKe5j&M0$M_yB36V+Gr4 zyFRX*WsG4k%Y4Vaol^Th4WH3o07E>+INc0vYYSF)|FVZM3{LBsDCm;a+RJcl`(j}I z%D}7WDx2{qz5)t|{AnU4n9bjS)MEF5)E9!GD+#t_47ppN6&bi&c}POQNZk-0z!tj? ztb6_K9UQ2W?7J9P_uc&`Yw;CpBjQQ93>xB~l&pzJxna~n6S%2?w~?INz`1u4U2)m& zlP}QXe60>nJc5j?k3=PSESzF!M6^}m#QWGJ)|NGc~pdV@nlN<{uwyZliH(WS#SwnnX3h8`dki+1L zj%iX0<*tQ$hSlq|!*CCn%*iR6&wg2IDGrlR zF2EWzJ$Ue3ROpoP47Tz7U6>Uf;Z=bW+nc!YZQ^X<9iEQ2lvm@V#^7G@X;{Q8jqMEb zYjy%tc8Bz(q|uidqX_rTIEUPnl@*V_3^sEv+5A*eJ#LX&&WC9;9cEix`U5X)U}X!@ z%URIm9;$^?HO{Ib}r3ze)_hGAc>L#SrA0_f>KPXgU-21QLyHja#QNf%MJovL43-|chNLVZAz zCw99fitU$3wLAA(w0sTkzWOymO{VQ1LsvJ{*L2!`2f4lVed7VU0(QJ#;3FSHTW6ml zu;$7wQ*@_Yx(OKmJ1qeCOzS28Z_Q0Rb8?OD-+LJ%yl9O>zY-%jRY^^k|GrkVymG@o zU$}M_pDpp{g;!#+FKCDrs>XxP(1$+Wl0pl0&@V#P^YOqYM)Sf8eT8@UCiyur@KWh% zL`NL5;0|AZ;-DE&X#7{iL64`%M1H3Ggpg>)@;i~7ghZp&GM&DfNUWhx2-S^47(C(; zHkz?Trqf_U+(_7HW`&Y)#wKzR!%qljj9H;1oH1tgJ`r9q6v2t)B%Cp3^*-T@P2?hm zpAgO%vwEL!#+VgK!Wm;y?-R}#vqDKYV+^(YM2RJkF#;L;4-_C&H=(*GrvTw~Gpl`x zG$kH7JkeAQeL{HMIE2BKPC|9#k_Mai2(O#)x+kXq0hmlqBYIUusBU_COs?mMBnnH5SRrl$9n=-3JCtN-u9t6KFw=S@>tSRToG zs(0EQX5laO0%*m z@g8y(!u)cwA*sUT!Wf~}{}gOY&_3pDY!1>jEx*|>NsH=hpjq{Mt%FiHIUCF~Dlh%s zSp}RVNBiB!@2@vuqbTy&36cv_U6YpKx7K5{CW$F)bcOf-Uelu5wNzinj=dXk8*?$5 zm~9L2r`Ainj$Dj(Og))%_9VHfuGstcdPLm<{tSrx&AwP1Q`VH^1@klLS&r&hH`Iz1 zbykExGh_)ArACNI2sE=wlo06gLdQTJ;VT*;(2NN*fdL5%JyB0F{DiR3Olo~j!a_6I zw3BeTnbZ1HNQBFc6+KT-_G4rvTyBOs9>azbG4y1y{upx&KM*lAlUkpMp~rC$W7GhC zOn_p{X?+$N0g7R$b_PTwKrtq@KA8Z;m<*2yPz>Kq#Wn~8D27cJJW(Y;u}M9NaWYGQ zVoZid1SmF+gC>uJ^_m1I1sp^N`PXFbo@kW5}+6^ zE8%E20u*C1JR(4`$vlbip4$IyP;8xcZ{BDx0OF#?e^#i3wVr5S7=J=o>;KGLLrRyh z)`YdjaZAvQL?GVdEh=NSArNmST{;5sX4a)65N~F&58^5%W8Z0l9Uu^I=CnRh2Fk3~ zClK!mLN|#WAj&`|A<+0bWFm%UQtPu4F*G{#nVj8-7@Bb~G?AJF;?1O?BoOb(#gJaj z{oh8s36y)H*yDUcSZI3Iom`v#wuO#8rgC!%Bw`kPe7{Ph$7rh1<;4(13$gM!aIcj7 zy=Fb_+;)XWG@txlKPU}0N`uf|qu=YkRV!TU``z#I@AZT7<+Ll*d=~qA-M4bIa|D}& z#ec6K6j`v=*FVrylUev;7_Yy2a$Sk`U%%H6in=yOP*g6-k6B$h%#bou&WBu$2CAm2kP46iGJ1<))Lk$^9}ShMt^8bZU-9#L!FvPDBjNr0+_A zVssKWIlB>{*yJ>#S91g?HqOn}3&&l}Ki6y_s>c4SULyc9CdHGL0LUhh$MIep0g#QS$RrAm z0LYkiG6;a|zv}xD$`SzCBr-T24k7@u@f4X%sSyAf0gz3~Ga76&8|xO)O!yl`S>R7| Mz3!j!Yj>RbKdyBw3jhEB literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 277786aa8ce..1b5473934bd 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -42,6 +42,7 @@ export enum Providers { VolcEngine = "VolcEngine", Voyage = "Voyage AI", xAI = "xAI", + SAP = "SAP Generative AI Hub", } export const provider_map: Record = { @@ -87,6 +88,7 @@ export const provider_map: Record = { DeepInfra: "deepinfra", Hosted_Vllm: "hosted_vllm", Infinity: "infinity", + SAP: "sap", }; const asset_logos_folder = "../ui/assets/logos/"; @@ -134,6 +136,7 @@ export const providerLogoMap: Record = { [Providers.JinaAI]: `${asset_logos_folder}jina.png`, [Providers.VolcEngine]: `${asset_logos_folder}volcengine.png`, [Providers.DeepInfra]: `${asset_logos_folder}deepinfra.png`, + [Providers.SAP]: `${asset_logos_folder}sap.png`, }; export const getProviderLogoAndName = (providerValue: string): { logo: string; displayName: string } => { From 0f63cbea5932206e091d93cc26123ac4e8e6ec7f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Dec 2025 22:30:22 +0530 Subject: [PATCH 040/388] [Feat] Interactions API - allow using all litellm providers (interactions -> responses api bridge) (#18373) * add BaseInteractionsTest * add interactions_api_handler * init bridge * init LiteLLMResponsesInteractionsConfig * LiteLLMResponsesInteractionsHandler * mv test * fixes api spec * docs * fix transform+iterators * docs fix * fix iterator --- docs/my-website/docs/interactions.md | 57 +++- .../__init__.py | 16 + .../handler.py | 156 +++++++++ .../streaming_iterator.py | 260 +++++++++++++++ .../transformation.py | 277 ++++++++++++++++ litellm/interactions/main.py | 24 +- provider_endpoints_support.json | 297 ++++++++++++------ .../interactions/base_interactions_test.py | 111 +++++++ .../interactions/test_gemini_interactions.py | 24 ++ .../test_litellm_responses_bridge.py | 29 ++ .../base_interactions_test.py | 113 +++++++ .../test_gemini_interactions.py | 24 ++ .../test_litellm_responses_bridge.py | 29 ++ 13 files changed, 1311 insertions(+), 106 deletions(-) create mode 100644 litellm/interactions/litellm_responses_transformation/__init__.py create mode 100644 litellm/interactions/litellm_responses_transformation/handler.py create mode 100644 litellm/interactions/litellm_responses_transformation/streaming_iterator.py create mode 100644 litellm/interactions/litellm_responses_transformation/transformation.py create mode 100644 tests/test_litellm/interactions/base_interactions_test.py create mode 100644 tests/test_litellm/interactions/test_gemini_interactions.py create mode 100644 tests/test_litellm/interactions/test_litellm_responses_bridge.py create mode 100644 tests/unified_google_tests/base_interactions_test.py create mode 100644 tests/unified_google_tests/test_gemini_interactions.py create mode 100644 tests/unified_google_tests/test_litellm_responses_bridge.py diff --git a/docs/my-website/docs/interactions.md b/docs/my-website/docs/interactions.md index 5458a4463f5..1cd0f7be867 100644 --- a/docs/my-website/docs/interactions.md +++ b/docs/my-website/docs/interactions.md @@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem'; | Logging | ✅ | Works across all integrations | | Streaming | ✅ | | | Loadbalancing | ✅ | Between supported models | -| Supported Providers | `gemini` | [Google Interactions API](https://ai.google.dev/gemini-api/docs/interactions) | +| Supported LLM providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai` etc. | ## **LiteLLM Python SDK Usage** @@ -207,8 +207,63 @@ for chunk in client.interactions.create_stream( } ``` +## **Calling non-Interactions API endpoints (`/interactions` to `/responses` Bridge)** + +LiteLLM allows you to call non-Interactions API models via a bridge to LiteLLM's `/responses` endpoint. This is useful for calling OpenAI, Anthropic, and other providers that don't natively support the Interactions API. + +#### Python SDK Usage + +```python showLineNumbers title="SDK Usage" +import litellm +import os + +# Set API key +os.environ["OPENAI_API_KEY"] = "your-openai-api-key" + +# Non-streaming interaction +response = litellm.interactions.create( + model="gpt-4o", + input="Tell me a short joke about programming." +) + +print(response.outputs[-1].text) +``` + +#### LiteLLM Proxy Usage + +**Setup Config:** + +```yaml showLineNumbers title="Example Configuration" +model_list: +- model_name: openai-model + litellm_params: + model: gpt-4o + api_key: os.environ/OPENAI_API_KEY +``` + +**Start Proxy:** + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +**Make Request:** + +```bash showLineNumbers title="non-Interactions API Model Request" +curl http://localhost:4000/v1beta/interactions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "openai-model", + "input": "Tell me a short joke about programming." + }' +``` + ## **Supported Providers** | Provider | Link to Usage | |----------|---------------| | Google AI Studio | [Usage](#quick-start) | +| All other LiteLLM providers | [Bridge Usage](#calling-non-interactions-api-endpoints-interactions-to-responses-bridge) | diff --git a/litellm/interactions/litellm_responses_transformation/__init__.py b/litellm/interactions/litellm_responses_transformation/__init__.py new file mode 100644 index 00000000000..2450a9f3d20 --- /dev/null +++ b/litellm/interactions/litellm_responses_transformation/__init__.py @@ -0,0 +1,16 @@ +""" +Bridge module for connecting Interactions API to Responses API via litellm.responses(). +""" + +from litellm.interactions.litellm_responses_transformation.handler import ( + LiteLLMResponsesInteractionsHandler, +) +from litellm.interactions.litellm_responses_transformation.transformation import ( + LiteLLMResponsesInteractionsConfig, +) + +__all__ = [ + "LiteLLMResponsesInteractionsHandler", + "LiteLLMResponsesInteractionsConfig", # Transformation config class (not BaseInteractionsAPIConfig) +] + diff --git a/litellm/interactions/litellm_responses_transformation/handler.py b/litellm/interactions/litellm_responses_transformation/handler.py new file mode 100644 index 00000000000..c2df8f96eff --- /dev/null +++ b/litellm/interactions/litellm_responses_transformation/handler.py @@ -0,0 +1,156 @@ +""" +Handler for transforming interactions API requests to litellm.responses requests. +""" + +from typing import ( + Any, + AsyncIterator, + Coroutine, + Dict, + Iterator, + Optional, + Union, + cast, +) + +import litellm +from litellm.interactions.litellm_responses_transformation.streaming_iterator import ( + LiteLLMResponsesInteractionsStreamingIterator, +) +from litellm.interactions.litellm_responses_transformation.transformation import ( + LiteLLMResponsesInteractionsConfig, +) +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator +from litellm.types.interactions import ( + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIResponse, + InteractionsAPIStreamingResponse, +) +from litellm.types.llms.openai import ResponsesAPIResponse + + +class LiteLLMResponsesInteractionsHandler: + """Handler for bridging Interactions API to Responses API via litellm.responses().""" + + def interactions_api_handler( + self, + model: str, + input: Optional[InteractionInput], + optional_params: InteractionsAPIOptionalRequestParams, + custom_llm_provider: Optional[str] = None, + _is_async: bool = False, + stream: Optional[bool] = None, + **kwargs, + ) -> Union[ + InteractionsAPIResponse, + Iterator[InteractionsAPIStreamingResponse], + Coroutine[ + Any, + Any, + Union[ + InteractionsAPIResponse, + AsyncIterator[InteractionsAPIStreamingResponse], + ], + ], + ]: + """ + Handle Interactions API request by calling litellm.responses(). + + Args: + model: The model to use + input: The input content + optional_params: Optional parameters for the request + custom_llm_provider: Override LLM provider + _is_async: Whether this is an async call + stream: Whether to stream the response + **kwargs: Additional parameters + + Returns: + InteractionsAPIResponse or streaming iterator + """ + # Transform interactions request to responses request + responses_request = ( + LiteLLMResponsesInteractionsConfig.transform_interactions_request_to_responses_request( + model=model, + input=input, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + stream=stream, + **kwargs, + ) + ) + + if _is_async: + return self.async_interactions_api_handler( + responses_request=responses_request, + model=model, + input=input, + optional_params=optional_params, + **kwargs, + ) + + # Call litellm.responses() + # Note: litellm.responses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] + # but the type checker may see it as a coroutine in some contexts + responses_response = litellm.responses( + **responses_request, + ) + + # Handle streaming response + if isinstance(responses_response, BaseResponsesAPIStreamingIterator): + return LiteLLMResponsesInteractionsStreamingIterator( + model=model, + litellm_custom_stream_wrapper=responses_response, + request_input=input, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_metadata=kwargs.get("litellm_metadata", {}), + ) + + # At this point, responses_response must be ResponsesAPIResponse (not streaming) + # Cast to satisfy type checker since we've already checked it's not a streaming iterator + responses_api_response = cast(ResponsesAPIResponse, responses_response) + + # Transform responses response to interactions response + return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( + responses_response=responses_api_response, + model=model, + ) + + async def async_interactions_api_handler( + self, + responses_request: Dict[str, Any], + model: str, + input: Optional[InteractionInput], + optional_params: InteractionsAPIOptionalRequestParams, + **kwargs, + ) -> Union[InteractionsAPIResponse, AsyncIterator[InteractionsAPIStreamingResponse]]: + """Async handler for interactions API requests.""" + # Call litellm.aresponses() + # Note: litellm.aresponses() returns Union[ResponsesAPIResponse, BaseResponsesAPIStreamingIterator] + responses_response = await litellm.aresponses( + **responses_request, + ) + + # Handle streaming response + if isinstance(responses_response, BaseResponsesAPIStreamingIterator): + return LiteLLMResponsesInteractionsStreamingIterator( + model=model, + litellm_custom_stream_wrapper=responses_response, + request_input=input, + optional_params=optional_params, + custom_llm_provider=responses_request.get("custom_llm_provider"), + litellm_metadata=kwargs.get("litellm_metadata", {}), + ) + + # At this point, responses_response must be ResponsesAPIResponse (not streaming) + # Cast to satisfy type checker since we've already checked it's not a streaming iterator + responses_api_response = cast(ResponsesAPIResponse, responses_response) + + # Transform responses response to interactions response + return LiteLLMResponsesInteractionsConfig.transform_responses_response_to_interactions_response( + responses_response=responses_api_response, + model=model, + ) + diff --git a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py new file mode 100644 index 00000000000..511b69e83b2 --- /dev/null +++ b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py @@ -0,0 +1,260 @@ +""" +Streaming iterator for transforming Responses API stream to Interactions API stream. +""" + +from typing import Any, AsyncIterator, Dict, Iterator, Optional, cast + +from litellm.responses.streaming_iterator import ( + BaseResponsesAPIStreamingIterator, + ResponsesAPIStreamingIterator, + SyncResponsesAPIStreamingIterator, +) +from litellm.types.interactions import ( + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIStreamingResponse, +) +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponseCreatedEvent, + ResponseInProgressEvent, + ResponsesAPIStreamingResponse, +) + + +class LiteLLMResponsesInteractionsStreamingIterator: + """ + Iterator that wraps Responses API streaming and transforms chunks to Interactions API format. + + This class handles both sync and async iteration, transforming Responses API + streaming events (output.text.delta, response.completed, etc.) to Interactions + API streaming events (content.delta, interaction.complete, etc.). + """ + + def __init__( + self, + model: str, + litellm_custom_stream_wrapper: BaseResponsesAPIStreamingIterator, + request_input: Optional[InteractionInput], + optional_params: InteractionsAPIOptionalRequestParams, + custom_llm_provider: Optional[str] = None, + litellm_metadata: Optional[Dict[str, Any]] = None, + ): + self.model = model + self.responses_stream_iterator = litellm_custom_stream_wrapper + self.request_input = request_input + self.optional_params = optional_params + self.custom_llm_provider = custom_llm_provider + self.litellm_metadata = litellm_metadata or {} + self.finished = False + self.collected_text = "" + self.sent_interaction_start = False + self.sent_content_start = False + + def _transform_responses_chunk_to_interactions_chunk( + self, + responses_chunk: ResponsesAPIStreamingResponse, + ) -> Optional[InteractionsAPIStreamingResponse]: + """ + Transform a Responses API streaming chunk to an Interactions API streaming chunk. + + Responses API events: + - output.text.delta -> content.delta + - response.completed -> interaction.complete + + Interactions API events: + - interaction.start + - content.start + - content.delta + - content.stop + - interaction.complete + """ + if not responses_chunk: + return None + + # Handle OutputTextDeltaEvent -> content.delta + if isinstance(responses_chunk, OutputTextDeltaEvent): + delta_text = responses_chunk.delta if isinstance(responses_chunk.delta, str) else "" + self.collected_text += delta_text + + # Send interaction.start if not sent + if not self.sent_interaction_start: + self.sent_interaction_start = True + return InteractionsAPIStreamingResponse( + event_type="interaction.start", + id=getattr(responses_chunk, "item_id", None) or f"interaction_{id(self)}", + object="interaction", + status="in_progress", + model=self.model, + ) + + # Send content.start if not sent + if not self.sent_content_start: + self.sent_content_start = True + return InteractionsAPIStreamingResponse( + event_type="content.start", + id=getattr(responses_chunk, "item_id", None), + object="content", + delta={"type": "text", "text": ""}, + ) + + # Send content.delta + return InteractionsAPIStreamingResponse( + event_type="content.delta", + id=getattr(responses_chunk, "item_id", None), + object="content", + delta={"text": delta_text}, + ) + + # Handle ResponseCreatedEvent or ResponseInProgressEvent -> interaction.start + if isinstance(responses_chunk, (ResponseCreatedEvent, ResponseInProgressEvent)): + if not self.sent_interaction_start: + self.sent_interaction_start = True + response_id = getattr(responses_chunk.response, "id", None) if hasattr(responses_chunk, "response") else None + return InteractionsAPIStreamingResponse( + event_type="interaction.start", + id=response_id or f"interaction_{id(self)}", + object="interaction", + status="in_progress", + model=self.model, + ) + + # Handle ResponseCompletedEvent -> interaction.complete + if isinstance(responses_chunk, ResponseCompletedEvent): + self.finished = True + response = responses_chunk.response + + # Send content.stop first if content was started + if self.sent_content_start: + # Note: We'll send this in the iterator, not here + pass + + # Send interaction.complete + return InteractionsAPIStreamingResponse( + event_type="interaction.complete", + id=getattr(response, "id", None) or f"interaction_{id(self)}", + object="interaction", + status="completed", + model=self.model, + outputs=[ + { + "type": "text", + "text": self.collected_text, + } + ], + ) + + # For other event types, return None (skip) + return None + + def __iter__(self) -> Iterator[InteractionsAPIStreamingResponse]: + """Sync iterator implementation.""" + return self + + def __next__(self) -> InteractionsAPIStreamingResponse: + """Get next chunk in sync mode.""" + if self.finished: + raise StopIteration + + # Check if we have a pending interaction.complete to send + if hasattr(self, "_pending_interaction_complete"): + pending: InteractionsAPIStreamingResponse = getattr(self, "_pending_interaction_complete") + delattr(self, "_pending_interaction_complete") + return pending + + # Use a loop instead of recursion to avoid stack overflow + sync_iterator = cast(SyncResponsesAPIStreamingIterator, self.responses_stream_iterator) + while True: + try: + # Get next chunk from responses API stream + chunk = next(sync_iterator) + + # Transform chunk (chunk is already a ResponsesAPIStreamingResponse) + transformed = self._transform_responses_chunk_to_interactions_chunk(chunk) + + if transformed: + # If we finished and content was started, send content.stop before interaction.complete + if self.finished and self.sent_content_start and transformed.event_type == "interaction.complete": + # Send content.stop first + content_stop = InteractionsAPIStreamingResponse( + event_type="content.stop", + id=transformed.id, + object="content", + delta={"type": "text", "text": self.collected_text}, + ) + # Store the interaction.complete to send next + self._pending_interaction_complete = transformed + return content_stop + return transformed + + # If no transformation, continue to next chunk (loop continues) + + except StopIteration: + self.finished = True + + # Send final events if needed + if self.sent_content_start: + return InteractionsAPIStreamingResponse( + event_type="content.stop", + object="content", + delta={"type": "text", "text": self.collected_text}, + ) + + raise StopIteration + + def __aiter__(self) -> AsyncIterator[InteractionsAPIStreamingResponse]: + """Async iterator implementation.""" + return self + + async def __anext__(self) -> InteractionsAPIStreamingResponse: + """Get next chunk in async mode.""" + if self.finished: + raise StopAsyncIteration + + # Check if we have a pending interaction.complete to send + if hasattr(self, "_pending_interaction_complete"): + pending: InteractionsAPIStreamingResponse = getattr(self, "_pending_interaction_complete") + delattr(self, "_pending_interaction_complete") + return pending + + # Use a loop instead of recursion to avoid stack overflow + async_iterator = cast(ResponsesAPIStreamingIterator, self.responses_stream_iterator) + while True: + try: + # Get next chunk from responses API stream + chunk = await async_iterator.__anext__() + + # Transform chunk (chunk is already a ResponsesAPIStreamingResponse) + transformed = self._transform_responses_chunk_to_interactions_chunk(chunk) + + if transformed: + # If we finished and content was started, send content.stop before interaction.complete + if self.finished and self.sent_content_start and transformed.event_type == "interaction.complete": + # Send content.stop first + content_stop = InteractionsAPIStreamingResponse( + event_type="content.stop", + id=transformed.id, + object="content", + delta={"type": "text", "text": self.collected_text}, + ) + # Store the interaction.complete to send next + self._pending_interaction_complete = transformed + return content_stop + return transformed + + # If no transformation, continue to next chunk (loop continues) + + except StopAsyncIteration: + self.finished = True + + # Send final events if needed + if self.sent_content_start: + return InteractionsAPIStreamingResponse( + event_type="content.stop", + object="content", + delta={"type": "text", "text": self.collected_text}, + ) + + raise StopAsyncIteration + diff --git a/litellm/interactions/litellm_responses_transformation/transformation.py b/litellm/interactions/litellm_responses_transformation/transformation.py new file mode 100644 index 00000000000..24b2c5dbde7 --- /dev/null +++ b/litellm/interactions/litellm_responses_transformation/transformation.py @@ -0,0 +1,277 @@ +""" +Transformation utilities for bridging Interactions API to Responses API. + +This module handles transforming between: +- Interactions API format (Google's format with Turn[], system_instruction, etc.) +- Responses API format (OpenAI's format with input[], instructions, etc.) +""" + +from typing import Any, Dict, List, Optional, cast + +from litellm.types.interactions import ( + InteractionInput, + InteractionsAPIOptionalRequestParams, + InteractionsAPIResponse, + Turn, +) +from litellm.types.llms.openai import ( + ResponseInputParam, + ResponsesAPIResponse, +) + + +class LiteLLMResponsesInteractionsConfig: + """Configuration class for transforming between Interactions API and Responses API.""" + + @staticmethod + def transform_interactions_request_to_responses_request( + model: str, + input: Optional[InteractionInput], + optional_params: InteractionsAPIOptionalRequestParams, + **kwargs, + ) -> Dict[str, Any]: + """ + Transform an Interactions API request to a Responses API request. + + Key transformations: + - system_instruction -> instructions + - input (string | Turn[]) -> input (ResponseInputParam) + - tools -> tools (similar format) + - generation_config -> temperature, top_p, etc. + """ + responses_request: Dict[str, Any] = { + "model": model, + } + + # Transform input + if input is not None: + responses_request["input"] = ( + LiteLLMResponsesInteractionsConfig._transform_interactions_input_to_responses_input( + input + ) + ) + + # Transform system_instruction -> instructions + if optional_params.get("system_instruction"): + responses_request["instructions"] = optional_params["system_instruction"] + + # Transform tools (similar format, pass through for now) + if optional_params.get("tools"): + responses_request["tools"] = optional_params["tools"] + + # Transform generation_config to temperature, top_p, etc. + generation_config = optional_params.get("generation_config") + if generation_config: + if isinstance(generation_config, dict): + if "temperature" in generation_config: + responses_request["temperature"] = generation_config["temperature"] + if "top_p" in generation_config: + responses_request["top_p"] = generation_config["top_p"] + if "top_k" in generation_config: + # Responses API doesn't have top_k, skip it + pass + if "max_output_tokens" in generation_config: + responses_request["max_output_tokens"] = generation_config["max_output_tokens"] + + # Pass through other optional params that match + passthrough_params = ["stream", "store", "metadata", "user"] + for param in passthrough_params: + if param in optional_params and optional_params[param] is not None: + responses_request[param] = optional_params[param] + + # Add any extra kwargs + responses_request.update(kwargs) + + return responses_request + + @staticmethod + def _transform_interactions_input_to_responses_input( + input: InteractionInput, + ) -> ResponseInputParam: + """ + Transform Interactions API input to Responses API input format. + + Interactions API input can be: + - string: "Hello" + - Turn[]: [{"role": "user", "content": [...]}] + - Content object + + Responses API input is: + - string: "Hello" + - Message[]: [{"role": "user", "content": [...]}] + """ + if isinstance(input, str): + # ResponseInputParam accepts str + return cast(ResponseInputParam, input) + + if isinstance(input, list): + # Turn[] format - convert to Responses API Message[] format + messages = [] + for turn in input: + if isinstance(turn, dict): + role = turn.get("role", "user") + content = turn.get("content", []) + + # Transform content array + transformed_content = ( + LiteLLMResponsesInteractionsConfig._transform_content_array(content) + ) + + messages.append({ + "role": role, + "content": transformed_content, + }) + elif isinstance(turn, Turn): + # Pydantic model + role = turn.role if hasattr(turn, "role") else "user" + content = turn.content if hasattr(turn, "content") else [] + + # Ensure content is a list for _transform_content_array + # Cast to List[Any] to handle various content types + if isinstance(content, list): + content_list: List[Any] = list(content) + elif content is not None: + content_list = [content] + else: + content_list = [] + + transformed_content = ( + LiteLLMResponsesInteractionsConfig._transform_content_array(content_list) + ) + + messages.append({ + "role": role, + "content": transformed_content, + }) + + return cast(ResponseInputParam, messages) + + # Single content object - wrap in message + if isinstance(input, dict): + return cast(ResponseInputParam, [{ + "role": "user", + "content": LiteLLMResponsesInteractionsConfig._transform_content_array( + input.get("content", []) if isinstance(input.get("content"), list) else [input] + ), + }]) + + # Fallback: convert to string + return cast(ResponseInputParam, str(input)) + + @staticmethod + def _transform_content_array(content: List[Any]) -> List[Dict[str, Any]]: + """Transform Interactions API content array to Responses API format.""" + if not isinstance(content, list): + # Single content item - wrap in array + content = [content] + + transformed: List[Dict[str, Any]] = [] + for item in content: + if isinstance(item, dict): + # Already in dict format, pass through + transformed.append(item) + elif isinstance(item, str): + # Plain string - wrap in text format + transformed.append({"type": "text", "text": item}) + else: + # Pydantic model or other - convert to dict + if hasattr(item, "model_dump"): + dumped = item.model_dump() + if isinstance(dumped, dict): + transformed.append(dumped) + else: + # Fallback: wrap in text format + transformed.append({"type": "text", "text": str(dumped)}) + elif hasattr(item, "dict"): + dumped = item.dict() + if isinstance(dumped, dict): + transformed.append(dumped) + else: + # Fallback: wrap in text format + transformed.append({"type": "text", "text": str(dumped)}) + else: + # Fallback: wrap in text format + transformed.append({"type": "text", "text": str(item)}) + + return transformed + + @staticmethod + def transform_responses_response_to_interactions_response( + responses_response: ResponsesAPIResponse, + model: Optional[str] = None, + ) -> InteractionsAPIResponse: + """ + Transform a Responses API response to an Interactions API response. + + Key transformations: + - Extract text from output[].content[].text + - Convert created_at (int) to created (ISO string) + - Map status + - Extract usage + """ + # Extract text from outputs + outputs = [] + if hasattr(responses_response, "output") and responses_response.output: + for output_item in responses_response.output: + # Use getattr with None default to safely access content + content = getattr(output_item, "content", None) + if content is not None: + content_items = content if isinstance(content, list) else [content] + for content_item in content_items: + # Check if content_item has text attribute + text = getattr(content_item, "text", None) + if text is not None: + outputs.append({ + "type": "text", + "text": text, + }) + elif isinstance(content_item, dict) and content_item.get("type") == "text": + outputs.append(content_item) + + # Convert created_at to ISO string + created_at = getattr(responses_response, "created_at", None) + if isinstance(created_at, int): + from datetime import datetime + created = datetime.fromtimestamp(created_at).isoformat() + elif created_at is not None and hasattr(created_at, "isoformat"): + created = created_at.isoformat() + else: + created = None + + # Map status + status = getattr(responses_response, "status", "completed") + if status == "completed": + interactions_status = "completed" + elif status == "in_progress": + interactions_status = "in_progress" + else: + interactions_status = status + + # Build interactions response + interactions_response_dict: Dict[str, Any] = { + "id": getattr(responses_response, "id", ""), + "object": "interaction", + "status": interactions_status, + "outputs": outputs, + "model": model or getattr(responses_response, "model", ""), + "created": created, + } + + # Add usage if available + # Map Responses API usage (input_tokens, output_tokens) to Interactions API spec format + # (total_input_tokens, total_output_tokens) + usage = getattr(responses_response, "usage", None) + if usage: + interactions_response_dict["usage"] = { + "total_input_tokens": getattr(usage, "input_tokens", 0), + "total_output_tokens": getattr(usage, "output_tokens", 0), + } + + # Add role + interactions_response_dict["role"] = "model" + + # Add updated (same as created for now) + interactions_response_dict["updated"] = created + + return InteractionsAPIResponse(**interactions_response_dict) + diff --git a/litellm/interactions/main.py b/litellm/interactions/main.py index 9fb58fc73d6..fb811b25b2f 100644 --- a/litellm/interactions/main.py +++ b/litellm/interactions/main.py @@ -272,18 +272,30 @@ def create( model=model, ) - if interactions_api_config is None: - raise ValueError( - f"Interactions API is not supported for provider: {custom_llm_provider}. " - "Currently only 'gemini' is supported." - ) - # Get optional params using utility (similar to responses API pattern) local_vars.update(kwargs) optional_params = InteractionsAPIRequestUtils.get_requested_interactions_api_optional_params( local_vars ) + # Check if this is a bridge provider (litellm_responses) - similar to responses API + # Either provider is explicitly "litellm_responses" or no config found (bridge to responses) + if custom_llm_provider == "litellm_responses" or interactions_api_config is None: + # Bridge to litellm.responses() for non-native providers + from litellm.interactions.litellm_responses_transformation.handler import ( + LiteLLMResponsesInteractionsHandler, + ) + handler = LiteLLMResponsesInteractionsHandler() + return handler.interactions_api_handler( + model=model or "", + input=input, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + _is_async=_is_async, + stream=stream, + **kwargs, + ) + litellm_logging_obj.update_environment_variables( model=model, optional_params=dict(optional_params), diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 82b48b67195..45ee47c01bc 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -47,7 +47,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "ai21": { @@ -64,7 +65,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "ai21_chat": { @@ -81,7 +83,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "amazon_nova": { @@ -98,7 +101,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "anthropic": { @@ -116,7 +120,8 @@ "batches": true, "rerank": false, "skills": true, - "a2a": true + "a2a": true, + "interactions": true } }, "anthropic_text": { @@ -134,7 +139,8 @@ "batches": true, "rerank": false, "skills": true, - "a2a": true + "a2a": true, + "interactions": true } }, "apertis": { @@ -167,7 +173,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "auto_router": { @@ -184,7 +191,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "bedrock": { @@ -201,7 +209,8 @@ "moderations": false, "batches": false, "rerank": true, - "a2a": true + "a2a": true, + "interactions": true } }, "sagemaker": { @@ -218,7 +227,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "aws_polly": { @@ -251,7 +261,8 @@ "moderations": true, "batches": true, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "azure_ai": { @@ -269,7 +280,8 @@ "batches": true, "rerank": false, "ocr": true, - "a2a": true + "a2a": true, + "interactions": true } }, "azure_ai/doc-intelligence": { @@ -303,7 +315,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "azure_text": { @@ -320,7 +333,8 @@ "moderations": true, "batches": true, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "baseten": { @@ -337,7 +351,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "bytez": { @@ -354,7 +369,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "cerebras": { @@ -371,7 +387,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "chutes": { @@ -404,7 +421,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "cloudflare": { @@ -421,7 +439,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "codestral": { @@ -438,7 +457,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "cohere": { @@ -455,7 +475,8 @@ "moderations": false, "batches": false, "rerank": true, - "a2a": true + "a2a": true, + "interactions": true } }, "cohere_chat": { @@ -472,7 +493,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "cometapi": { @@ -489,7 +511,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "compactifai": { @@ -506,7 +529,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "custom": { @@ -523,7 +547,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "custom_openai": { @@ -540,7 +565,8 @@ "moderations": true, "batches": true, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "dashscope": { @@ -557,7 +583,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "databricks": { @@ -574,7 +601,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "dataforseo": { @@ -608,7 +636,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "deepgram": { @@ -625,7 +654,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "deepinfra": { @@ -642,7 +672,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "deepseek": { @@ -659,7 +690,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "elevenlabs": { @@ -676,7 +708,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "exa_ai": { @@ -710,7 +743,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "fal_ai": { @@ -727,7 +761,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "featherless_ai": { @@ -744,7 +779,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "fireworks_ai": { @@ -761,7 +797,8 @@ "moderations": false, "batches": false, "rerank": true, - "a2a": true + "a2a": true, + "interactions": true } }, "firecrawl": { @@ -812,7 +849,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "galadriel": { @@ -829,7 +867,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "github_copilot": { @@ -846,7 +885,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "github": { @@ -863,7 +903,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "vertex_ai": { @@ -881,7 +922,8 @@ "batches": false, "rerank": false, "ocr": true, - "a2a": true + "a2a": true, + "interactions": true } }, "vertex_ai/chirp": { @@ -932,7 +974,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "groq": { @@ -949,7 +992,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "heroku": { @@ -966,7 +1010,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "hosted_vllm": { @@ -984,7 +1029,8 @@ "batches": true, "files": true, "rerank": true, - "a2a": true + "a2a": true, + "interactions": true } }, "huggingface": { @@ -1001,7 +1047,8 @@ "moderations": false, "batches": false, "rerank": true, - "a2a": true + "a2a": true, + "interactions": true } }, "hyperbolic": { @@ -1018,7 +1065,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "watsonx": { @@ -1035,7 +1083,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "infinity": { @@ -1084,7 +1133,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "lemonade": { @@ -1101,7 +1151,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "litellm_proxy": { @@ -1118,7 +1169,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "llamafile": { @@ -1135,7 +1187,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "lm_studio": { @@ -1152,7 +1205,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "maritalk": { @@ -1169,7 +1223,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "meta_llama": { @@ -1186,7 +1241,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "mistral": { @@ -1204,7 +1260,8 @@ "batches": false, "rerank": false, "ocr": true, - "a2a": true + "a2a": true, + "interactions": true } }, "moonshot": { @@ -1221,7 +1278,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "docker_model_runner": { @@ -1238,7 +1296,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "morph": { @@ -1255,7 +1314,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "nanogpt": { @@ -1288,7 +1348,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "nlp_cloud": { @@ -1305,7 +1366,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "novita": { @@ -1322,7 +1384,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "nscale": { @@ -1339,7 +1402,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "nvidia_nim": { @@ -1356,7 +1420,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "oci": { @@ -1373,7 +1438,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "ollama": { @@ -1390,7 +1456,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "ollama_chat": { @@ -1407,7 +1474,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "oobabooga": { @@ -1424,7 +1492,8 @@ "moderations": true, "batches": true, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "openai": { @@ -1450,7 +1519,8 @@ "retrieve_container_file": true, "retrieve_container_file_content": true, "delete_container_file": true, - "a2a": true + "a2a": true, + "interactions": true } }, "openai_like": { @@ -1483,7 +1553,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "ovhcloud": { @@ -1500,7 +1571,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "parallel_ai": { @@ -1535,7 +1607,8 @@ "batches": false, "rerank": false, "search": true, - "a2a": true + "a2a": true, + "interactions": true } }, "petals": { @@ -1552,7 +1625,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "poe": { @@ -1585,7 +1659,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "predibase": { @@ -1602,7 +1677,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "recraft": { @@ -1635,7 +1711,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "runwayml": { @@ -1669,7 +1746,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "searxng": { @@ -1703,7 +1781,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "sap": { @@ -1720,7 +1799,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "snowflake": { @@ -1737,7 +1817,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "synthetic": { @@ -1770,7 +1851,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "text-completion-openai": { @@ -1787,7 +1869,8 @@ "moderations": true, "batches": true, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "together_ai": { @@ -1804,7 +1887,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "topaz": { @@ -1821,7 +1905,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "tavily": { @@ -1855,7 +1940,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "v0": { @@ -1872,7 +1958,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "vercel_ai_gateway": { @@ -1889,7 +1976,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "vllm": { @@ -1907,7 +1995,8 @@ "batches": true, "files": true, "rerank": true, - "a2a": true + "a2a": true, + "interactions": true } }, "volcengine": { @@ -1924,7 +2013,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "voyage": { @@ -1957,7 +2047,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "watsonx_text": { @@ -1974,7 +2065,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "xai": { @@ -1991,7 +2083,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "xinference": { @@ -2024,7 +2117,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "ragflow": { @@ -2042,7 +2136,8 @@ "batches": false, "rerank": false, "vector_stores": true, - "a2a": true + "a2a": true, + "interactions": true } }, "cursor": { @@ -2059,7 +2154,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "langgraph": { @@ -2076,7 +2172,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "vertex_ai/agent_engine": { @@ -2093,7 +2190,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } }, "pydantic_ai_agents": { @@ -2144,7 +2242,8 @@ "moderations": false, "batches": false, "rerank": false, - "a2a": true + "a2a": true, + "interactions": true } } } diff --git a/tests/test_litellm/interactions/base_interactions_test.py b/tests/test_litellm/interactions/base_interactions_test.py new file mode 100644 index 00000000000..b7748a45f32 --- /dev/null +++ b/tests/test_litellm/interactions/base_interactions_test.py @@ -0,0 +1,111 @@ +""" +Abstract base class for Interactions API tests. + +This class provides common test cases that can be inherited by provider-specific +test classes. Subclasses must implement get_model() and get_api_key(). +""" + +import os +from abc import ABC, abstractmethod + +import pytest + +import litellm.interactions as interactions + + +class BaseInteractionsTest(ABC): + """Abstract base class for interactions API tests. + + Subclasses must implement get_model() and get_api_key(). + All test methods are inherited and run against the specific provider. + """ + + @abstractmethod + def get_model(self) -> str: + """Return the model string for this provider.""" + pass + + @abstractmethod + def get_api_key(self) -> str: + """Return the API key for this provider.""" + pass + + def test_create_simple_string_input(self): + """Test creating an interaction with a simple string input.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response = interactions.create( + model=self.get_model(), + input="Hello, what is 2 + 2?", + api_key=api_key, + ) + assert response is not None + assert response.id is not None or response.status is not None + + # Check outputs per OpenAPI spec + if response.outputs: + assert len(response.outputs) > 0 + + # Check usage per OpenAPI spec + if response.usage: + # Usage is a dict in InteractionsAPIResponse + if isinstance(response.usage, dict): + assert response.usage.get("input_tokens") is not None or response.usage.get("output_tokens") is not None + else: + # If it's an object, check attributes + assert hasattr(response.usage, "input_tokens") or hasattr(response.usage, "output_tokens") + + def test_create_with_system_instruction(self): + """Test creating an interaction with system_instruction.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response = interactions.create( + model=self.get_model(), + input="What are you?", + system_instruction="You are a helpful pirate assistant. Always respond like a pirate.", + api_key=api_key, + ) + assert response is not None + # Verify the response reflects the system instruction + if response.outputs: + assert len(response.outputs) > 0 + + def test_create_streaming(self): + """Test creating a streaming interaction.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response_stream = interactions.create( + model=self.get_model(), + input="Count from 1 to 3.", + stream=True, + api_key=api_key, + ) + + # Collect all chunks + chunks = [] + for chunk in response_stream: + chunks.append(chunk) + + assert len(chunks) > 0 + + @pytest.mark.asyncio + async def test_acreate_simple(self): + """Test async interaction creation.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response = await interactions.acreate( + model=self.get_model(), + input="What is the speed of light?", + api_key=api_key, + ) + assert response is not None + assert response.id is not None or response.status is not None + diff --git a/tests/test_litellm/interactions/test_gemini_interactions.py b/tests/test_litellm/interactions/test_gemini_interactions.py new file mode 100644 index 00000000000..c75e1d8a860 --- /dev/null +++ b/tests/test_litellm/interactions/test_gemini_interactions.py @@ -0,0 +1,24 @@ +""" +Tests for Gemini Interactions API. + +Inherits from BaseInteractionsTest to run the same test suite against Gemini. +""" + +import os + +from tests.test_litellm.interactions.base_interactions_test import ( + BaseInteractionsTest, +) + + +class TestGeminiInteractions(BaseInteractionsTest): + """Test Gemini Interactions API using the base test suite.""" + + def get_model(self) -> str: + """Return the Gemini model string.""" + return "gemini/gemini-2.5-flash" + + def get_api_key(self) -> str: + """Return the Gemini API key from environment.""" + return os.getenv("GEMINI_API_KEY", "") + diff --git a/tests/test_litellm/interactions/test_litellm_responses_bridge.py b/tests/test_litellm/interactions/test_litellm_responses_bridge.py new file mode 100644 index 00000000000..f99090f8363 --- /dev/null +++ b/tests/test_litellm/interactions/test_litellm_responses_bridge.py @@ -0,0 +1,29 @@ +""" +Tests for LiteLLM Responses bridge provider. + +Inherits from BaseInteractionsTest to run the same test suite against +the litellm_responses bridge provider, which calls litellm.responses() internally. +""" + +import os + +from tests.test_litellm.interactions.base_interactions_test import ( + BaseInteractionsTest, +) + + +class TestLiteLLMResponsesBridge(BaseInteractionsTest): + """Test LiteLLM Responses bridge using the base test suite.""" + + def get_model(self) -> str: + """Return the model string for the bridge provider. + + The bridge provider uses litellm.responses() internally, so we can + use any model that litellm.responses() supports (e.g., gpt-4o). + """ + return "gpt-4o" + + def get_api_key(self) -> str: + """Return the OpenAI API key from environment.""" + return os.getenv("OPENAI_API_KEY", "") + diff --git a/tests/unified_google_tests/base_interactions_test.py b/tests/unified_google_tests/base_interactions_test.py new file mode 100644 index 00000000000..0a07fe87fa5 --- /dev/null +++ b/tests/unified_google_tests/base_interactions_test.py @@ -0,0 +1,113 @@ +""" +Abstract base class for Interactions API tests. + +This class provides common test cases that can be inherited by provider-specific +test classes. Subclasses must implement get_model() and get_api_key(). +""" + +import os +from abc import ABC, abstractmethod + +import pytest +import litellm +import litellm.interactions as interactions + + +class BaseInteractionsTest(ABC): + """Abstract base class for interactions API tests. + + Subclasses must implement get_model() and get_api_key(). + All test methods are inherited and run against the specific provider. + """ + + @abstractmethod + def get_model(self) -> str: + """Return the model string for this provider.""" + pass + + @abstractmethod + def get_api_key(self) -> str: + """Return the API key for this provider.""" + pass + + def test_create_simple_string_input(self): + """Test creating an interaction with a simple string input.""" + litellm._turn_on_debug() + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response = interactions.create( + model=self.get_model(), + input="Hello, what is 2 + 2?", + api_key=api_key, + ) + assert response is not None + assert response.id is not None or response.status is not None + + # Check outputs per OpenAPI spec + if response.outputs: + assert len(response.outputs) > 0 + + # Check usage per OpenAPI spec + # The spec defines: total_input_tokens, total_output_tokens + if response.usage: + # Usage is a dict in InteractionsAPIResponse + if isinstance(response.usage, dict): + assert response.usage.get("total_input_tokens") is not None or response.usage.get("total_output_tokens") is not None + else: + # If it's an object, check attributes + assert hasattr(response.usage, "total_input_tokens") or hasattr(response.usage, "total_output_tokens") + + def test_create_with_system_instruction(self): + """Test creating an interaction with system_instruction.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response = interactions.create( + model=self.get_model(), + input="What are you?", + system_instruction="You are a helpful pirate assistant. Always respond like a pirate.", + api_key=api_key, + ) + assert response is not None + # Verify the response reflects the system instruction + if response.outputs: + assert len(response.outputs) > 0 + + def test_create_streaming(self): + """Test creating a streaming interaction.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response_stream = interactions.create( + model=self.get_model(), + input="Count from 1 to 3.", + stream=True, + api_key=api_key, + ) + + # Collect all chunks + chunks = [] + for chunk in response_stream: + chunks.append(chunk) + + assert len(chunks) > 0 + + @pytest.mark.asyncio + async def test_acreate_simple(self): + """Test async interaction creation.""" + api_key = self.get_api_key() + if not api_key: + pytest.skip(f"API key not set for {self.__class__.__name__}") + + response = await interactions.acreate( + model=self.get_model(), + input="What is the speed of light?", + api_key=api_key, + ) + assert response is not None + assert response.id is not None or response.status is not None + diff --git a/tests/unified_google_tests/test_gemini_interactions.py b/tests/unified_google_tests/test_gemini_interactions.py new file mode 100644 index 00000000000..eb1e104d80f --- /dev/null +++ b/tests/unified_google_tests/test_gemini_interactions.py @@ -0,0 +1,24 @@ +""" +Tests for Gemini Interactions API. + +Inherits from BaseInteractionsTest to run the same test suite against Gemini. +""" + +import os + +from tests.unified_google_tests.base_interactions_test import ( + BaseInteractionsTest, +) + + +class TestGeminiInteractions(BaseInteractionsTest): + """Test Gemini Interactions API using the base test suite.""" + + def get_model(self) -> str: + """Return the Gemini model string.""" + return "gemini/gemini-2.5-flash" + + def get_api_key(self) -> str: + """Return the Gemini API key from environment.""" + return os.getenv("GEMINI_API_KEY", "") + diff --git a/tests/unified_google_tests/test_litellm_responses_bridge.py b/tests/unified_google_tests/test_litellm_responses_bridge.py new file mode 100644 index 00000000000..3c1342f650c --- /dev/null +++ b/tests/unified_google_tests/test_litellm_responses_bridge.py @@ -0,0 +1,29 @@ +""" +Tests for LiteLLM Responses bridge provider. + +Inherits from BaseInteractionsTest to run the same test suite against +the litellm_responses bridge provider, which calls litellm.responses() internally. +""" + +import os + +from tests.unified_google_tests.base_interactions_test import ( + BaseInteractionsTest, +) + + +class TestLiteLLMResponsesBridge(BaseInteractionsTest): + """Test LiteLLM Responses bridge using the base test suite.""" + + def get_model(self) -> str: + """Return the model string for the bridge provider. + + The bridge provider uses litellm.responses() internally, so we can + use any model that litellm.responses() supports (e.g., gpt-4o). + """ + return "gpt-4o" + + def get_api_key(self) -> str: + """Return the OpenAI API key from environment.""" + return os.getenv("OPENAI_API_KEY", "") + From 4f7682f6706171620042c34311fc6ed3633006fa Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Dec 2025 22:30:37 +0530 Subject: [PATCH 041/388] [Feat] RAG query endpoint - Add RAG Search / Query endpoint (#18376) * add rag query to llm api endpoints * add rag query as a new endpoint * docs - new endpoint * ingest+query * docs add rag query --- docs/my-website/docs/rag_ingest.md | 34 ++- docs/my-website/docs/rag_query.md | 273 +++++++++++++++++++++++ docs/my-website/sidebars.js | 9 +- litellm/proxy/_types.py | 2 + litellm/proxy/rag_endpoints/endpoints.py | 149 ++++++++++++- 5 files changed, 460 insertions(+), 7 deletions(-) create mode 100644 docs/my-website/docs/rag_query.md diff --git a/docs/my-website/docs/rag_ingest.md b/docs/my-website/docs/rag_ingest.md index 536151febdc..1133b85f206 100644 --- a/docs/my-website/docs/rag_ingest.md +++ b/docs/my-website/docs/rag_ingest.md @@ -4,9 +4,13 @@ All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector | Feature | Supported | |---------|-----------| -| Logging | ✅ | +| Logging | Yes | | Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` | +:::tip +After ingesting documents, use [/rag/query](./rag_query.md) to search and generate responses with your ingested content. +::: + ## Quick Start ### OpenAI @@ -82,9 +86,33 @@ curl -X POST "http://localhost:4000/v1/rag/ingest" \ } ``` -## Query the Vector Store +## Query with RAG -After ingestion, query with `/vector_stores/{vector_store_id}/search`: +After ingestion, use the [/rag/query](./rag_query.md) endpoint to search and generate LLM responses: + +```bash showLineNumbers title="RAG Query" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is the main topic?"}], + "retrieval_config": { + "vector_store_id": "vs_xyz789", + "custom_llm_provider": "openai", + "top_k": 5 + } + }' +``` + +This will: +1. Search the vector store for relevant context +2. Prepend the context to your messages +3. Generate an LLM response + +### Direct Vector Store Search + +Alternatively, search the vector store directly with `/vector_stores/{vector_store_id}/search`: ```bash showLineNumbers title="Search the vector store" curl -X POST "http://localhost:4000/v1/vector_stores/vs_xyz789/search" \ diff --git a/docs/my-website/docs/rag_query.md b/docs/my-website/docs/rag_query.md new file mode 100644 index 00000000000..2ae030880d6 --- /dev/null +++ b/docs/my-website/docs/rag_query.md @@ -0,0 +1,273 @@ +# /rag/query + +RAG Query endpoint: **Search Vector Store → (Rerank) → LLM Completion** + +| Feature | Supported | +|---------|-----------| +| Logging | Yes | +| Streaming | Yes | +| Reranking | Yes (optional) | +| Supported Providers | `openai`, `bedrock`, `vertex_ai` | + +## Quick Start + +```bash showLineNumbers title="RAG Query with OpenAI" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "vs_abc123", + "custom_llm_provider": "openai", + "top_k": 5 + } + }' +``` + +## How It Works + +The RAG query endpoint performs the following steps: + +1. **Extract Query**: Extracts the query text from the last user message +2. **Search Vector Store**: Searches the specified vector store for relevant context +3. **Rerank (Optional)**: Reranks the search results using a reranking model +4. **Generate Response**: Calls the LLM with the retrieved context prepended to the messages + +## Response + +The response follows the standard OpenAI chat completion format, with additional search metadata: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1703123456, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "LiteLLM is a unified interface for 100+ LLMs..." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 150, + "completion_tokens": 50, + "total_tokens": 200 + }, + "_hidden_params": { + "search_results": {...}, + "rerank_results": {...} + } +} +``` + +## With Reranking + +Add a `rerank` configuration to improve result quality: + +```bash showLineNumbers title="RAG Query with Reranking" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "vs_abc123", + "custom_llm_provider": "openai", + "top_k": 10 + }, + "rerank": { + "enabled": true, + "model": "cohere/rerank-english-v3.0", + "top_n": 3 + } + }' +``` + +## Streaming + +Enable streaming for real-time responses: + +```bash showLineNumbers title="RAG Query with Streaming" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "vs_abc123", + "custom_llm_provider": "openai" + }, + "stream": true + }' +``` + +## Request Parameters + +### Top-Level + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `model` | string | Yes | The LLM model to use for generation | +| `messages` | array | Yes | Array of chat messages (OpenAI format) | +| `retrieval_config` | object | Yes | Vector store search configuration | +| `rerank` | object | No | Reranking configuration | +| `stream` | boolean | No | Enable streaming (default: `false`) | + +### retrieval_config + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `vector_store_id` | string | **required** | ID of the vector store to search | +| `custom_llm_provider` | string | `"openai"` | Vector store provider | +| `top_k` | integer | `10` | Number of results to retrieve | + +### rerank + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `enabled` | boolean | `false` | Enable reranking | +| `model` | string | - | Reranking model (e.g., `cohere/rerank-english-v3.0`) | +| `top_n` | integer | `5` | Number of results after reranking | + +## End-to-End Example + +### 1. Ingest a Document + +First, ingest a document using the [/rag/ingest](./rag_ingest.md) endpoint: + +```bash showLineNumbers title="Step 1: Ingest" +curl -X POST "http://localhost:4000/v1/rag/ingest" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d "{ + \"file\": { + \"filename\": \"company_docs.txt\", + \"content\": \"$(base64 -i company_docs.txt)\", + \"content_type\": \"text/plain\" + }, + \"ingest_options\": { + \"vector_store\": { + \"custom_llm_provider\": \"openai\" + } + } + }" +``` + +Response: +```json +{ + "id": "ingest_abc123", + "status": "completed", + "vector_store_id": "vs_xyz789", + "file_id": "file-123" +} +``` + +### 2. Query with RAG + +Now query the ingested documents: + +```bash showLineNumbers title="Step 2: Query" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "What products does the company offer?"} + ], + "retrieval_config": { + "vector_store_id": "vs_xyz789", + "custom_llm_provider": "openai", + "top_k": 5 + } + }' +``` + +Response: +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Based on the company documents, the company offers..." + }, + "finish_reason": "stop" + } + ] +} +``` + +## Provider Examples + +### Bedrock + +```bash showLineNumbers title="RAG Query with Bedrock" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "KNOWLEDGE_BASE_ID", + "custom_llm_provider": "bedrock", + "top_k": 5 + } + }' +``` + +### Vertex AI + +```bash showLineNumbers title="RAG Query with Vertex AI" +curl -X POST "http://localhost:4000/v1/rag/query" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "vertex_ai/gemini-1.5-pro", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "your-corpus-id", + "custom_llm_provider": "vertex_ai", + "top_k": 5 + } + }' +``` + +## Python SDK + +```python showLineNumbers title="Using litellm.aquery()" +import litellm + +response = await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is LiteLLM?"}], + retrieval_config={ + "vector_store_id": "vs_abc123", + "custom_llm_provider": "openai", + "top_k": 5, + }, + rerank={ + "enabled": True, + "model": "cohere/rerank-english-v3.0", + "top_n": 3, + }, +) + +print(response.choices[0].message.content) +``` + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index ae63bade025..948014a792b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -528,7 +528,14 @@ const sidebars = { "proxy/pass_through_guardrails" ] }, - "rag_ingest", + { + type: "category", + label: "/rag", + items: [ + "rag_ingest", + "rag_query", + ] + }, "realtime", "rerank", "response_api", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 06067035c18..e29d4301744 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -388,6 +388,8 @@ class LiteLLMRoutes(enum.Enum): litellm_native_routes = [ "/rag/ingest", "/v1/rag/ingest", + "/rag/query", + "/v1/rag/query", ] anthropic_routes = [ diff --git a/litellm/proxy/rag_endpoints/endpoints.py b/litellm/proxy/rag_endpoints/endpoints.py index c0b5103f47f..79b4fd6873d 100644 --- a/litellm/proxy/rag_endpoints/endpoints.py +++ b/litellm/proxy/rag_endpoints/endpoints.py @@ -1,8 +1,9 @@ """ -RAG Ingest Endpoints for LiteLLM Proxy. +RAG Endpoints for LiteLLM Proxy. -Provides an all-in-one API for document ingestion: -Upload -> (OCR) -> Chunk -> Embed -> Vector Store +Provides: +- /rag/ingest: All-in-one document ingestion pipeline (Upload -> Chunk -> Embed -> Vector Store) +- /rag/query: RAG query pipeline (Search -> Rerank -> LLM Completion) """ import base64 @@ -198,3 +199,145 @@ async def rag_ingest( status_code=500, detail={"error": str(e)}, ) + + +@router.post( + "/v1/rag/query", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["rag"], +) +@router.post( + "/rag/query", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["rag"], +) +async def rag_query( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + RAG Query endpoint - search vector store, optionally rerank, and generate LLM response. + + This endpoint: + 1. Extracts the query from the last user message + 2. Searches the vector store for relevant context + 3. Optionally reranks the results + 4. Generates an LLM response with the retrieved context + + ## Example Request: + ```bash + curl -X POST "http://localhost:4000/v1/rag/query" \\ + -H "Authorization: Bearer sk-1234" \\ + -H "Content-Type: application/json" \\ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "vs_abc123", + "custom_llm_provider": "openai", + "top_k": 5 + } + }' + ``` + + ## With Reranking: + ```bash + curl -X POST "http://localhost:4000/v1/rag/query" \\ + -H "Authorization: Bearer sk-1234" \\ + -H "Content-Type: application/json" \\ + -d '{ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "What is LiteLLM?"}], + "retrieval_config": { + "vector_store_id": "vs_abc123", + "custom_llm_provider": "openai", + "top_k": 10 + }, + "rerank": { + "enabled": true, + "model": "cohere/rerank-english-v3.0", + "top_n": 3 + } + }' + ``` + """ + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + llm_router, + proxy_config, + version, + ) + + try: + # Parse request body + data = await _read_request_body(request) + + # Extract required fields + model = data.get("model") + messages = data.get("messages") + retrieval_config = data.get("retrieval_config") + rerank = data.get("rerank") + stream = data.get("stream", False) + + # Validate required fields + if not model: + raise HTTPException( + status_code=400, + detail={"error": "model is required"}, + ) + if not messages: + raise HTTPException( + status_code=400, + detail={"error": "messages is required"}, + ) + if not retrieval_config: + raise HTTPException( + status_code=400, + detail={"error": "retrieval_config is required"}, + ) + if "vector_store_id" not in retrieval_config: + raise HTTPException( + status_code=400, + detail={"error": "retrieval_config must contain 'vector_store_id'"}, + ) + + # Add litellm data + request_data: Dict[str, Any] = {} + request_data = await add_litellm_data_to_request( + data=request_data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + verbose_proxy_logger.debug( + f"RAG Query - model: {model}, retrieval_config: {retrieval_config}" + ) + + # Call query + response = await litellm.aquery( + model=model, + messages=messages, + retrieval_config=retrieval_config, + rerank=rerank, + stream=stream, + router=llm_router, + **request_data, + ) + + return response + + except HTTPException: + raise + except Exception as e: + verbose_proxy_logger.exception(f"RAG Query failed: {e}") + raise HTTPException( + status_code=500, + detail={"error": str(e)}, + ) From 1b8cb31f4eaa672c9fca41607aece7181ad97b35 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Tue, 23 Dec 2025 10:24:18 -0800 Subject: [PATCH 042/388] [Refactor] Consolidate lazy import handlers with registry pattern (#18389) --- .gitignore | 1 + litellm/__init__.py | 97 +-- litellm/_lazy_imports.py | 1109 ++++++----------------------- litellm/_lazy_imports_registry.py | 305 ++++++++ 4 files changed, 556 insertions(+), 956 deletions(-) create mode 100644 litellm/_lazy_imports_registry.py diff --git a/.gitignore b/.gitignore index aa973201fd1..8196d1d9f24 100644 --- a/.gitignore +++ b/.gitignore @@ -100,3 +100,4 @@ update_model_cost_map.py tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py litellm/proxy/_experimental/out/guardrails/index.html scripts/test_vertex_ai_search.py +LAZY_LOADING_IMPROVEMENTS.md diff --git a/litellm/__init__.py b/litellm/__init__.py index dfcee0e2d3c..9da367c3672 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1574,93 +1574,26 @@ if TYPE_CHECKING: def __getattr__(name: str) -> Any: - """Lazy import handler""" - from ._lazy_imports import ( - COST_CALCULATOR_NAMES, - LITELLM_LOGGING_NAMES, - UTILS_NAMES, - TOKEN_COUNTER_NAMES, - LLM_CLIENT_CACHE_NAMES, - BEDROCK_TYPES_NAMES, - TYPES_UTILS_NAMES, - CACHING_NAMES, - HTTP_HANDLER_NAMES, - DOTPROMPT_NAMES, - LLM_CONFIG_NAMES, - TYPES_NAMES, - ) + """Lazy import handler with cached registry for improved performance.""" + # Use cached registry from _lazy_imports instead of importing tuples every time + from ._lazy_imports import _get_lazy_import_registry - # Lazy load cost_calculator functions - if name in COST_CALCULATOR_NAMES: - from ._lazy_imports import _lazy_import_cost_calculator - return _lazy_import_cost_calculator(name) - - # Lazy load litellm_logging functions - if name in LITELLM_LOGGING_NAMES: - from ._lazy_imports import _lazy_import_litellm_logging - return _lazy_import_litellm_logging(name) - - # Lazy load utils functions - if name in UTILS_NAMES: - from ._lazy_imports import _lazy_import_utils - return _lazy_import_utils(name) + registry = _get_lazy_import_registry() - # Lazy load token counter utilities - if name in TOKEN_COUNTER_NAMES: - from ._lazy_imports import _lazy_import_token_counter - return _lazy_import_token_counter(name) - - # Lazy load Bedrock type aliases - if name in BEDROCK_TYPES_NAMES: - from ._lazy_imports import _lazy_import_bedrock_types - return _lazy_import_bedrock_types(name) - - # Lazy load common types.utils symbols - if name in TYPES_UTILS_NAMES: - from ._lazy_imports import _lazy_import_types_utils - return _lazy_import_types_utils(name) - - # Lazy load LLM client cache and its singleton - if name in LLM_CLIENT_CACHE_NAMES: - from ._lazy_imports import _lazy_import_llm_client_cache - return _lazy_import_llm_client_cache(name) - - # Lazy load caching classes - if name in CACHING_NAMES: - from ._lazy_imports import _lazy_import_caching - return _lazy_import_caching(name) - - # Lazy-load HTTP handler singletons used across the codebase - if name in HTTP_HANDLER_NAMES: - from ._lazy_imports import _lazy_import_http_handlers - - return _lazy_import_http_handlers(name) - - # Lazy load dotprompt integration globals - if name in DOTPROMPT_NAMES: - from ._lazy_imports import _lazy_import_dotprompt - - return _lazy_import_dotprompt(name) - - # Lazy load LLM config classes - if name in LLM_CONFIG_NAMES: - from ._lazy_imports import _lazy_import_llm_configs - - return _lazy_import_llm_configs(name) - - # Lazy load types - if name in TYPES_NAMES: - from ._lazy_imports import _lazy_import_types - - return _lazy_import_types(name) + # Check if name is in registry and call the cached handler function + if name in registry: + handler_func = registry[name] + return handler_func(name) # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": - from .main import encoding as _encoding - # Cache it in the module's __dict__ for subsequent accesses - import sys - sys.modules[__name__].__dict__["encoding"] = _encoding - return _encoding + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + # Check if already cached + if "encoding" not in _globals: + from .main import encoding as _encoding + _globals["encoding"] = _encoding + return _globals["encoding"] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 044fad924ac..c1b3e1df976 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -1,12 +1,66 @@ +""" +Lazy Import System + +This module implements lazy loading for LiteLLM attributes. Instead of importing +everything when the module loads, we only import things when they're actually used. + +How it works: +1. When someone accesses `litellm.some_attribute`, Python calls __getattr__ in __init__.py +2. __getattr__ looks up the attribute name in a registry +3. The registry points to a handler function (like _lazy_import_utils) +4. The handler function imports the module and returns the attribute +5. The result is cached so we don't import it again + +This makes importing litellm much faster because we don't load heavy dependencies +until they're actually needed. +""" +import importlib import sys -from typing import Any, Optional, cast +from typing import Any, Optional, cast, Callable + +# Import all the data structures that define what can be lazy-loaded +# These are just lists of names and maps of where to find them +from ._lazy_imports_registry import ( + # Name tuples + COST_CALCULATOR_NAMES, + LITELLM_LOGGING_NAMES, + UTILS_NAMES, + TOKEN_COUNTER_NAMES, + LLM_CLIENT_CACHE_NAMES, + BEDROCK_TYPES_NAMES, + TYPES_UTILS_NAMES, + CACHING_NAMES, + HTTP_HANDLER_NAMES, + DOTPROMPT_NAMES, + LLM_CONFIG_NAMES, + TYPES_NAMES, + # Import maps + _UTILS_IMPORT_MAP, + _COST_CALCULATOR_IMPORT_MAP, + _TYPES_UTILS_IMPORT_MAP, + _TOKEN_COUNTER_IMPORT_MAP, + _BEDROCK_TYPES_IMPORT_MAP, + _CACHING_IMPORT_MAP, + _LITELLM_LOGGING_IMPORT_MAP, + _DOTPROMPT_IMPORT_MAP, + _TYPES_IMPORT_MAP, + _LLM_CONFIGS_IMPORT_MAP, +) def _get_litellm_globals() -> dict: - """Helper to get the globals dictionary of the litellm module.""" + """ + Get the globals dictionary of the litellm module. + + This is where we cache imported attributes so we don't import them twice. + When you do `litellm.some_function`, it gets stored in this dictionary. + """ return sys.modules["litellm"].__dict__ -# Lazy loader for default encoding to avoid importing tiktoken at module import time +# These are special lazy loaders for things that are used internally +# They're separate from the main lazy import system because they have specific use cases + +# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup _default_encoding: Optional[Any] = None @@ -75,944 +129,251 @@ def _get_token_counter_new() -> Any: _token_counter_new_func = _token_counter_imported return _token_counter_new_func -# Cost calculator names that support lazy loading via _lazy_import_cost_calculator -COST_CALCULATOR_NAMES = ( - "completion_cost", - "cost_per_token", - "response_cost_calculator", -) -# Litellm logging names that support lazy loading via _lazy_import_litellm_logging -LITELLM_LOGGING_NAMES = ( - "Logging", - "modify_integration", -) +# ============================================================================ +# MAIN LAZY IMPORT SYSTEM +# ============================================================================ -# Utils names that support lazy loading via _lazy_import_utils -UTILS_NAMES = ( - "exception_type", "get_optional_params", "get_response_string", "token_counter", - "create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling", - "supports_web_search", "supports_url_context", "supports_response_schema", - "supports_parallel_function_calling", "supports_vision", "supports_audio_input", - "supports_audio_output", "supports_system_messages", "supports_reasoning", - "get_litellm_params", "acreate", "get_max_tokens", "get_model_info", - "register_prompt_template", "validate_environment", "check_valid_key", - "register_model", "encode", "decode", "_calculate_retry_after", "_should_retry", - "get_supported_openai_params", "get_api_base", "get_first_chars_messages", - "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", - "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", - "ModelResponseListIterator", "get_valid_models", -) +# This registry maps attribute names (like "ModelResponse") to handler functions +# It's built once the first time someone accesses a lazy-loaded attribute +# Example: {"ModelResponse": _lazy_import_utils, "Cache": _lazy_import_caching, ...} +_LAZY_IMPORT_REGISTRY: Optional[dict[str, Callable[[str], Any]]] = None -# Token counter names that support lazy loading via _lazy_import_token_counter -TOKEN_COUNTER_NAMES = ( - "get_modified_max_tokens", -) -# LLM client cache names that support lazy loading via _lazy_import_llm_client_cache -LLM_CLIENT_CACHE_NAMES = ( - "LLMClientCache", - "in_memory_llm_clients_cache", -) +def _get_lazy_import_registry() -> dict[str, Callable[[str], Any]]: + """ + Build the registry that maps attribute names to their handler functions. + + This is called once, the first time someone accesses a lazy-loaded attribute. + After that, we just look up the handler function in this dictionary. + + Returns: + Dictionary like {"ModelResponse": _lazy_import_utils, ...} + """ + global _LAZY_IMPORT_REGISTRY + if _LAZY_IMPORT_REGISTRY is None: + # Build the registry by going through each category and mapping + # all the names in that category to their handler function + _LAZY_IMPORT_REGISTRY = {} + # For each category, map all its names to the handler function + # Example: All names in UTILS_NAMES get mapped to _lazy_import_utils + for name in COST_CALCULATOR_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_cost_calculator + for name in LITELLM_LOGGING_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_litellm_logging + for name in UTILS_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils + for name in TOKEN_COUNTER_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_token_counter + for name in LLM_CLIENT_CACHE_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_client_cache + for name in BEDROCK_TYPES_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_bedrock_types + for name in TYPES_UTILS_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_types_utils + for name in CACHING_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_caching + for name in HTTP_HANDLER_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_http_handlers + for name in DOTPROMPT_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_dotprompt + for name in LLM_CONFIG_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_configs + for name in TYPES_NAMES: + _LAZY_IMPORT_REGISTRY[name] = _lazy_import_types + + return _LAZY_IMPORT_REGISTRY -# Bedrock type names that support lazy loading via _lazy_import_bedrock_types -BEDROCK_TYPES_NAMES = ( - "COHERE_EMBEDDING_INPUT_TYPES", -) -# Common types from litellm.types.utils that support lazy loading via -# _lazy_import_types_utils -TYPES_UTILS_NAMES = ( - "ImageObject", - "BudgetConfig", - "all_litellm_params", - "_litellm_completion_params", - "CredentialItem", - "PriorityReservationDict", - "StandardKeyGenerationConfig", - "SearchProviders", - "GenericStreamingChunk", -) - -# Caching / cache classes that support lazy loading via _lazy_import_caching -CACHING_NAMES = ( - "Cache", - "DualCache", - "RedisCache", - "InMemoryCache", -) - -# HTTP handler names that support lazy loading via _lazy_import_http_handlers -HTTP_HANDLER_NAMES = ( - "module_level_aclient", - "module_level_client", -) - -# Dotprompt integration names that support lazy loading via _lazy_import_dotprompt -DOTPROMPT_NAMES = ( - "global_prompt_manager", - "global_prompt_directory", - "set_global_prompt_directory", -) - -# LLM config classes that support lazy loading via _lazy_import_llm_configs -LLM_CONFIG_NAMES = ( - "AmazonConverseConfig", - "OpenAILikeChatConfig", - "GaladrielChatConfig", - "GithubChatConfig", - "AzureAnthropicConfig", - "BytezChatConfig", - "CompactifAIChatConfig", - "EmpowerChatConfig", - "MinimaxChatConfig", - "AiohttpOpenAIChatConfig", - "HuggingFaceChatConfig", - "HuggingFaceEmbeddingConfig", - "OobaboogaConfig", - "MaritalkConfig", - "OpenrouterConfig", - "DataRobotConfig", - "AnthropicConfig", - "AnthropicTextConfig", - "GroqSTTConfig", - "TritonConfig", - "TritonGenerateConfig", - "TritonInferConfig", - "TritonEmbeddingConfig", - "HuggingFaceRerankConfig", - "DatabricksConfig", - "DatabricksEmbeddingConfig", - "PredibaseConfig", - "ReplicateConfig", - "SnowflakeConfig", - "CohereRerankConfig", - "CohereRerankV2Config", - "AzureAIRerankConfig", - "InfinityRerankConfig", - "JinaAIRerankConfig", - "DeepinfraRerankConfig", - "HostedVLLMRerankConfig", - "NvidiaNimRerankConfig", - "NvidiaNimRankingConfig", - "VertexAIRerankConfig", - "FireworksAIRerankConfig", - "VoyageRerankConfig", - "ClarifaiConfig", -) - -# Types that support lazy loading via _lazy_import_types -TYPES_NAMES = ( - "GuardrailItem", -) - -# Lazy import for utils module - imports only the requested item by name. -# Note: PLR0915 (too many statements) is suppressed because the many if statements -# are intentional - each attribute is imported individually only when requested, -# ensuring true lazy imports rather than importing the entire utils module. -def _lazy_import_utils(name: str) -> Any: # noqa: PLR0915 - """Lazy import for utils module - imports only the requested item by name.""" +def _generic_lazy_import(name: str, import_map: dict[str, tuple[str, str]], category: str) -> Any: + """ + Generic function that handles lazy importing for most attributes. + + This is the workhorse function - it does the actual importing and caching. + Most handler functions just call this with their specific import map. + + Steps: + 1. Check if the name exists in the import map (if not, raise error) + 2. Check if we've already imported it (if yes, return cached value) + 3. Look up where to find it (module_path and attr_name from the map) + 4. Import the module (Python caches this automatically) + 5. Get the attribute from the module + 6. Cache it in _globals so we don't import again + 7. Return it + + Args: + name: The attribute name someone is trying to access (e.g., "ModelResponse") + import_map: Dictionary telling us where to find each attribute + Format: {"ModelResponse": (".utils", "ModelResponse")} + category: Just for error messages (e.g., "Utils", "Cost calculator") + """ + # Step 1: Make sure this attribute exists in our map + if name not in import_map: + raise AttributeError(f"{category} lazy import: unknown attribute {name!r}") + + # Step 2: Get the cache (where we store imported things) _globals = _get_litellm_globals() - if name == "exception_type": - from .utils import exception_type as _exception_type - _globals["exception_type"] = _exception_type - return _exception_type - if name == "get_optional_params": - from .utils import get_optional_params as _get_optional_params - _globals["get_optional_params"] = _get_optional_params - return _get_optional_params + # Step 3: If we've already imported it, just return the cached version + if name in _globals: + return _globals[name] - if name == "get_response_string": - from .utils import get_response_string as _get_response_string - _globals["get_response_string"] = _get_response_string - return _get_response_string + # Step 4: Look up where to find this attribute + # The map tells us: (module_path, attribute_name) + # Example: (".utils", "ModelResponse") means "look in .utils module, get ModelResponse" + module_path, attr_name = import_map[name] - if name == "token_counter": - from .utils import token_counter as _token_counter - _globals["token_counter"] = _token_counter - return _token_counter + # Step 5: Import the module + # Python automatically caches modules in sys.modules, so calling this twice is fast + # If module_path starts with ".", it's a relative import (needs package="litellm") + # Otherwise it's an absolute import (like "litellm.caching.caching") + if module_path.startswith("."): + module = importlib.import_module(module_path, package="litellm") + else: + module = importlib.import_module(module_path) - if name == "create_pretrained_tokenizer": - from .utils import create_pretrained_tokenizer as _create_pretrained_tokenizer - _globals["create_pretrained_tokenizer"] = _create_pretrained_tokenizer - return _create_pretrained_tokenizer + # Step 6: Get the actual attribute from the module + # Example: getattr(utils_module, "ModelResponse") returns the ModelResponse class + value = getattr(module, attr_name) - if name == "create_tokenizer": - from .utils import create_tokenizer as _create_tokenizer - _globals["create_tokenizer"] = _create_tokenizer - return _create_tokenizer + # Step 7: Cache it so we don't have to import again next time + _globals[name] = value - if name == "supports_function_calling": - from .utils import supports_function_calling as _supports_function_calling - _globals["supports_function_calling"] = _supports_function_calling - return _supports_function_calling - - if name == "supports_web_search": - from .utils import supports_web_search as _supports_web_search - _globals["supports_web_search"] = _supports_web_search - return _supports_web_search - - if name == "supports_url_context": - from .utils import supports_url_context as _supports_url_context - _globals["supports_url_context"] = _supports_url_context - return _supports_url_context - - if name == "supports_response_schema": - from .utils import supports_response_schema as _supports_response_schema - _globals["supports_response_schema"] = _supports_response_schema - return _supports_response_schema - - if name == "supports_parallel_function_calling": - from .utils import ( - supports_parallel_function_calling as _supports_parallel_function_calling, - ) - _globals["supports_parallel_function_calling"] = _supports_parallel_function_calling - return _supports_parallel_function_calling - - if name == "supports_vision": - from .utils import supports_vision as _supports_vision - _globals["supports_vision"] = _supports_vision - return _supports_vision - - if name == "supports_audio_input": - from .utils import supports_audio_input as _supports_audio_input - _globals["supports_audio_input"] = _supports_audio_input - return _supports_audio_input - - if name == "supports_audio_output": - from .utils import supports_audio_output as _supports_audio_output - _globals["supports_audio_output"] = _supports_audio_output - return _supports_audio_output - - if name == "supports_system_messages": - from .utils import supports_system_messages as _supports_system_messages - _globals["supports_system_messages"] = _supports_system_messages - return _supports_system_messages - - if name == "supports_reasoning": - from .utils import supports_reasoning as _supports_reasoning - _globals["supports_reasoning"] = _supports_reasoning - return _supports_reasoning - - if name == "get_litellm_params": - from .utils import get_litellm_params as _get_litellm_params - _globals["get_litellm_params"] = _get_litellm_params - return _get_litellm_params - - if name == "acreate": - from .utils import acreate as _acreate - _globals["acreate"] = _acreate - return _acreate - - if name == "get_max_tokens": - from .utils import get_max_tokens as _get_max_tokens - _globals["get_max_tokens"] = _get_max_tokens - return _get_max_tokens - - if name == "get_model_info": - from .utils import get_model_info as _get_model_info - _globals["get_model_info"] = _get_model_info - return _get_model_info - - if name == "register_prompt_template": - from .utils import register_prompt_template as _register_prompt_template - _globals["register_prompt_template"] = _register_prompt_template - return _register_prompt_template - - if name == "validate_environment": - from .utils import validate_environment as _validate_environment - _globals["validate_environment"] = _validate_environment - return _validate_environment - - if name == "check_valid_key": - from .utils import check_valid_key as _check_valid_key - _globals["check_valid_key"] = _check_valid_key - return _check_valid_key - - if name == "register_model": - from .utils import register_model as _register_model - _globals["register_model"] = _register_model - return _register_model - - if name == "encode": - from .utils import encode as _encode - _globals["encode"] = _encode - return _encode - - if name == "decode": - from .utils import decode as _decode - _globals["decode"] = _decode - return _decode - - if name == "_calculate_retry_after": - from .utils import _calculate_retry_after as __calculate_retry_after - _globals["_calculate_retry_after"] = __calculate_retry_after - return __calculate_retry_after - - if name == "_should_retry": - from .utils import _should_retry as __should_retry - _globals["_should_retry"] = __should_retry - return __should_retry - - if name == "get_supported_openai_params": - from .utils import get_supported_openai_params as _get_supported_openai_params - _globals["get_supported_openai_params"] = _get_supported_openai_params - return _get_supported_openai_params - - if name == "get_api_base": - from .utils import get_api_base as _get_api_base - _globals["get_api_base"] = _get_api_base - return _get_api_base - - if name == "get_first_chars_messages": - from .utils import get_first_chars_messages as _get_first_chars_messages - _globals["get_first_chars_messages"] = _get_first_chars_messages - return _get_first_chars_messages - - if name == "ModelResponse": - from .utils import ModelResponse as _ModelResponse - _globals["ModelResponse"] = _ModelResponse - return _ModelResponse - - if name == "ModelResponseStream": - from .utils import ModelResponseStream as _ModelResponseStream - _globals["ModelResponseStream"] = _ModelResponseStream - return _ModelResponseStream - - if name == "EmbeddingResponse": - from .utils import EmbeddingResponse as _EmbeddingResponse - _globals["EmbeddingResponse"] = _EmbeddingResponse - return _EmbeddingResponse - - if name == "ImageResponse": - from .utils import ImageResponse as _ImageResponse - _globals["ImageResponse"] = _ImageResponse - return _ImageResponse - - if name == "TranscriptionResponse": - from .utils import TranscriptionResponse as _TranscriptionResponse - _globals["TranscriptionResponse"] = _TranscriptionResponse - return _TranscriptionResponse - - if name == "TextCompletionResponse": - from .utils import TextCompletionResponse as _TextCompletionResponse - _globals["TextCompletionResponse"] = _TextCompletionResponse - return _TextCompletionResponse - - if name == "get_provider_fields": - from .utils import get_provider_fields as _get_provider_fields - _globals["get_provider_fields"] = _get_provider_fields - return _get_provider_fields - - if name == "ModelResponseListIterator": - from .utils import ModelResponseListIterator as _ModelResponseListIterator - _globals["ModelResponseListIterator"] = _ModelResponseListIterator - return _ModelResponseListIterator - - if name == "get_valid_models": - from .utils import get_valid_models as _get_valid_models - _globals["get_valid_models"] = _get_valid_models - return _get_valid_models - - raise AttributeError(f"Utils lazy import: unknown attribute {name!r}") + # Step 8: Return it + return value + + +# ============================================================================ +# HANDLER FUNCTIONS +# ============================================================================ +# These functions are called when someone accesses a lazy-loaded attribute. +# Most of them just call _generic_lazy_import with their specific import map. +# The registry (above) maps attribute names to these handler functions. + +def _lazy_import_utils(name: str) -> Any: + """Handler for utils module attributes (ModelResponse, token_counter, etc.)""" + return _generic_lazy_import(name, _UTILS_IMPORT_MAP, "Utils") def _lazy_import_cost_calculator(name: str) -> Any: - """Lazy import for cost_calculator functions.""" - _globals = _get_litellm_globals() - if name == "completion_cost": - from .cost_calculator import completion_cost as _completion_cost - _globals["completion_cost"] = _completion_cost - return _completion_cost - - if name == "cost_per_token": - from .cost_calculator import cost_per_token as _cost_per_token - _globals["cost_per_token"] = _cost_per_token - return _cost_per_token - - if name == "response_cost_calculator": - from .cost_calculator import ( - response_cost_calculator as _response_cost_calculator, - ) - _globals["response_cost_calculator"] = _response_cost_calculator - return _response_cost_calculator - - raise AttributeError(f"Cost calculator lazy import: unknown attribute {name!r}") + """Handler for cost calculator functions (completion_cost, cost_per_token, etc.)""" + return _generic_lazy_import(name, _COST_CALCULATOR_IMPORT_MAP, "Cost calculator") def _lazy_import_token_counter(name: str) -> Any: - """Lazy import for token_counter utilities.""" - _globals = _get_litellm_globals() - - if name == "get_modified_max_tokens": - from litellm.litellm_core_utils.token_counter import ( - get_modified_max_tokens as _get_modified_max_tokens, - ) - - _globals["get_modified_max_tokens"] = _get_modified_max_tokens - return _get_modified_max_tokens - - raise AttributeError(f"Token counter lazy import: unknown attribute {name!r}") + """Handler for token counter utilities""" + return _generic_lazy_import(name, _TOKEN_COUNTER_IMPORT_MAP, "Token counter") def _lazy_import_bedrock_types(name: str) -> Any: - """Lazy import for Bedrock type aliases.""" - _globals = _get_litellm_globals() - - if name == "COHERE_EMBEDDING_INPUT_TYPES": - from litellm.types.llms.bedrock import ( - COHERE_EMBEDDING_INPUT_TYPES as _COHERE_EMBEDDING_INPUT_TYPES, - ) - - _globals["COHERE_EMBEDDING_INPUT_TYPES"] = _COHERE_EMBEDDING_INPUT_TYPES - return _COHERE_EMBEDDING_INPUT_TYPES - - raise AttributeError(f"Bedrock types lazy import: unknown attribute {name!r}") + """Handler for Bedrock type aliases""" + return _generic_lazy_import(name, _BEDROCK_TYPES_IMPORT_MAP, "Bedrock types") def _lazy_import_types_utils(name: str) -> Any: - """Lazy import for common types and constants from litellm.types.utils.""" - _globals = _get_litellm_globals() - - if name == "ImageObject": - from .types.utils import ImageObject as _ImageObject - - _globals["ImageObject"] = _ImageObject - return _ImageObject - - if name == "BudgetConfig": - from .types.utils import BudgetConfig as _BudgetConfig - - _globals["BudgetConfig"] = _BudgetConfig - return _BudgetConfig - - if name == "all_litellm_params": - from .types.utils import all_litellm_params as _all_litellm_params - - _globals["all_litellm_params"] = _all_litellm_params - return _all_litellm_params - - if name == "_litellm_completion_params": - from .types.utils import all_litellm_params as _all_litellm_params - - _globals["_litellm_completion_params"] = _all_litellm_params - return _all_litellm_params - - if name == "CredentialItem": - from .types.utils import CredentialItem as _CredentialItem - - _globals["CredentialItem"] = _CredentialItem - return _CredentialItem - - if name == "PriorityReservationDict": - from .types.utils import PriorityReservationDict as _PriorityReservationDict - - _globals["PriorityReservationDict"] = _PriorityReservationDict - return _PriorityReservationDict - - if name == "StandardKeyGenerationConfig": - from .types.utils import ( - StandardKeyGenerationConfig as _StandardKeyGenerationConfig, - ) - - _globals["StandardKeyGenerationConfig"] = _StandardKeyGenerationConfig - return _StandardKeyGenerationConfig - - if name == "SearchProviders": - from .types.utils import SearchProviders as _SearchProviders - - _globals["SearchProviders"] = _SearchProviders - return _SearchProviders - - if name == "GenericStreamingChunk": - from .types.utils import GenericStreamingChunk as _GenericStreamingChunk - - _globals["GenericStreamingChunk"] = _GenericStreamingChunk - return _GenericStreamingChunk - - raise AttributeError(f"Types utils lazy import: unknown attribute {name!r}") + """Handler for types from litellm.types.utils (BudgetConfig, ImageObject, etc.)""" + return _generic_lazy_import(name, _TYPES_UTILS_IMPORT_MAP, "Types utils") def _lazy_import_caching(name: str) -> Any: - """Lazy import for caching module classes.""" - _globals = _get_litellm_globals() + """Handler for caching classes (Cache, DualCache, RedisCache, etc.)""" + return _generic_lazy_import(name, _CACHING_IMPORT_MAP, "Caching") - if name == "Cache": - from litellm.caching.caching import Cache as _Cache +def _lazy_import_dotprompt(name: str) -> Any: + """Handler for dotprompt integration globals""" + return _generic_lazy_import(name, _DOTPROMPT_IMPORT_MAP, "Dotprompt") - _globals["Cache"] = _Cache - return _Cache - if name == "DualCache": - from litellm.caching.caching import DualCache as _DualCache +def _lazy_import_types(name: str) -> Any: + """Handler for type classes (GuardrailItem, etc.)""" + return _generic_lazy_import(name, _TYPES_IMPORT_MAP, "Types") - _globals["DualCache"] = _DualCache - return _DualCache - if name == "RedisCache": - from litellm.caching.caching import RedisCache as _RedisCache +def _lazy_import_llm_configs(name: str) -> Any: + """Handler for LLM config classes (AnthropicConfig, OpenAILikeChatConfig, etc.)""" + return _generic_lazy_import(name, _LLM_CONFIGS_IMPORT_MAP, "LLM config") - _globals["RedisCache"] = _RedisCache - return _RedisCache - - if name == "InMemoryCache": - from litellm.caching.caching import InMemoryCache as _InMemoryCache - - _globals["InMemoryCache"] = _InMemoryCache - return _InMemoryCache - - raise AttributeError(f"Caching lazy import: unknown attribute {name!r}") +def _lazy_import_litellm_logging(name: str) -> Any: + """Handler for litellm_logging module (Logging, modify_integration)""" + return _generic_lazy_import(name, _LITELLM_LOGGING_IMPORT_MAP, "Litellm logging") +# ============================================================================ +# SPECIAL HANDLERS +# ============================================================================ +# These handlers have custom logic that doesn't fit the generic pattern def _lazy_import_llm_client_cache(name: str) -> Any: - """Lazy import for LLM client cache class and singleton.""" + """ + Handler for LLM client cache - has special logic for singleton instance. + + This one is different because: + - "LLMClientCache" is the class itself + - "in_memory_llm_clients_cache" is a singleton instance of that class + So we need custom logic to handle both cases. + """ _globals = _get_litellm_globals() - + + # If already cached, return it + if name in _globals: + return _globals[name] + + # Import the class + module = importlib.import_module("litellm.caching.llm_caching_handler") + LLMClientCache = getattr(module, "LLMClientCache") + + # If they want the class itself, return it if name == "LLMClientCache": - from litellm.caching.llm_caching_handler import ( - LLMClientCache as _LLMClientCache, - ) - - _globals["LLMClientCache"] = _LLMClientCache - return _LLMClientCache - + _globals["LLMClientCache"] = LLMClientCache + return LLMClientCache + + # If they want the singleton instance, create it (only once) if name == "in_memory_llm_clients_cache": - from litellm.caching.llm_caching_handler import ( - LLMClientCache as _LLMClientCache, - ) - - instance = _LLMClientCache() - # Only populate the requested singleton name to keep lazy-import - # semantics consistent with other helpers (no extra symbols). + instance = LLMClientCache() _globals["in_memory_llm_clients_cache"] = instance return instance - + raise AttributeError(f"LLM client cache lazy import: unknown attribute {name!r}") -def _lazy_import_litellm_logging(name: str) -> Any: - """Lazy import for litellm_logging module.""" - _globals = _get_litellm_globals() - if name == "Logging": - from litellm.litellm_core_utils.litellm_logging import Logging as _Logging - _globals["Logging"] = _Logging - return _Logging - - if name == "modify_integration": - from litellm.litellm_core_utils.litellm_logging import ( - modify_integration as _modify_integration, - ) - _globals["modify_integration"] = _modify_integration - return _modify_integration - - raise AttributeError(f"Litellm logging lazy import: unknown attribute {name!r}") - - def _lazy_import_http_handlers(name: str) -> Any: - """Lazy import and instantiate module-level HTTP handlers.""" + """ + Handler for HTTP clients - has special logic for creating client instances. + + This one is different because: + - These aren't just imports, they're actual client instances that need to be created + - They need configuration (timeout, etc.) from the module globals + - They use factory functions instead of direct instantiation + """ _globals = _get_litellm_globals() if name == "module_level_aclient": - # Use shared async client factory instead of directly instantiating AsyncHTTPHandler + # Create an async HTTP client using the factory function from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + # Get timeout from module config (if set) timeout = _globals.get("request_timeout") params = {"timeout": timeout, "client_alias": "module level aclient"} - # llm_provider is only used for cache keying; use a string identifier but - # cast to Any so static type checkers don't complain about the literal. + + # Create the client instance provider_id = cast(Any, "litellm_module_level_client") async_client = get_async_httpx_client( llm_provider=provider_id, params=params, ) + + # Cache it so we don't create it again _globals["module_level_aclient"] = async_client return async_client if name == "module_level_client": - # Import handler type locally to avoid heavy imports at module load time + # Create a sync HTTP client from litellm.llms.custom_httpx.http_handler import HTTPHandler timeout = _globals.get("request_timeout") sync_client = HTTPHandler(timeout=timeout) + + # Cache it _globals["module_level_client"] = sync_client return sync_client raise AttributeError(f"HTTP handlers lazy import: unknown attribute {name!r}") - - -def _lazy_import_dotprompt(name: str) -> Any: - """Lazy import for dotprompt integration globals.""" - _globals = _get_litellm_globals() - - if name == "global_prompt_manager": - from litellm.integrations.dotprompt import ( - global_prompt_manager as _global_prompt_manager, - ) - - _globals["global_prompt_manager"] = _global_prompt_manager - return _global_prompt_manager - - if name == "global_prompt_directory": - from litellm.integrations.dotprompt import ( - global_prompt_directory as _global_prompt_directory, - ) - - _globals["global_prompt_directory"] = _global_prompt_directory - return _global_prompt_directory - - if name == "set_global_prompt_directory": - from litellm.integrations.dotprompt import ( - set_global_prompt_directory as _set_global_prompt_directory, - ) - - _globals["set_global_prompt_directory"] = _set_global_prompt_directory - return _set_global_prompt_directory - - raise AttributeError(f"Dotprompt lazy import: unknown attribute {name!r}") - - -def _lazy_import_types(name: str) -> Any: - """Lazy import for type classes.""" - _globals = _get_litellm_globals() - - if name == "GuardrailItem": - from litellm.types.guardrails import GuardrailItem as _GuardrailItem - - _globals["GuardrailItem"] = _GuardrailItem - return _GuardrailItem - - raise AttributeError(f"Types lazy import: unknown attribute {name!r}") - - -def _lazy_import_llm_configs(name: str) -> Any: # noqa: PLR0915 - """Lazy import for LLM config classes.""" - _globals = _get_litellm_globals() - - if name == "AmazonConverseConfig": - from .llms.bedrock.chat.converse_transformation import ( - AmazonConverseConfig as _AmazonConverseConfig, - ) - - _globals["AmazonConverseConfig"] = _AmazonConverseConfig - return _AmazonConverseConfig - - if name == "OpenAILikeChatConfig": - from .llms.openai_like.chat.handler import ( - OpenAILikeChatConfig as _OpenAILikeChatConfig, - ) - - _globals["OpenAILikeChatConfig"] = _OpenAILikeChatConfig - return _OpenAILikeChatConfig - - if name == "GaladrielChatConfig": - from .llms.galadriel.chat.transformation import ( - GaladrielChatConfig as _GaladrielChatConfig, - ) - - _globals["GaladrielChatConfig"] = _GaladrielChatConfig - return _GaladrielChatConfig - - if name == "GithubChatConfig": - from .llms.github.chat.transformation import ( - GithubChatConfig as _GithubChatConfig, - ) - - _globals["GithubChatConfig"] = _GithubChatConfig - return _GithubChatConfig - - if name == "AzureAnthropicConfig": - from .llms.azure_ai.anthropic.transformation import ( - AzureAnthropicConfig as _AzureAnthropicConfig, - ) - - _globals["AzureAnthropicConfig"] = _AzureAnthropicConfig - return _AzureAnthropicConfig - - if name == "BytezChatConfig": - from .llms.bytez.chat.transformation import BytezChatConfig as _BytezChatConfig - - _globals["BytezChatConfig"] = _BytezChatConfig - return _BytezChatConfig - - if name == "CompactifAIChatConfig": - from .llms.compactifai.chat.transformation import ( - CompactifAIChatConfig as _CompactifAIChatConfig, - ) - - _globals["CompactifAIChatConfig"] = _CompactifAIChatConfig - return _CompactifAIChatConfig - - if name == "EmpowerChatConfig": - from .llms.empower.chat.transformation import ( - EmpowerChatConfig as _EmpowerChatConfig, - ) - - _globals["EmpowerChatConfig"] = _EmpowerChatConfig - return _EmpowerChatConfig - - if name == "MinimaxChatConfig": - from .llms.minimax.chat.transformation import ( - MinimaxChatConfig as _MinimaxChatConfig, - ) - - _globals["MinimaxChatConfig"] = _MinimaxChatConfig - return _MinimaxChatConfig - - if name == "AiohttpOpenAIChatConfig": - from .llms.aiohttp_openai.chat.transformation import ( - AiohttpOpenAIChatConfig as _AiohttpOpenAIChatConfig, - ) - - _globals["AiohttpOpenAIChatConfig"] = _AiohttpOpenAIChatConfig - return _AiohttpOpenAIChatConfig - - if name == "HuggingFaceChatConfig": - from .llms.huggingface.chat.transformation import ( - HuggingFaceChatConfig as _HuggingFaceChatConfig, - ) - - _globals["HuggingFaceChatConfig"] = _HuggingFaceChatConfig - return _HuggingFaceChatConfig - - if name == "HuggingFaceEmbeddingConfig": - from .llms.huggingface.embedding.transformation import ( - HuggingFaceEmbeddingConfig as _HuggingFaceEmbeddingConfig, - ) - - _globals["HuggingFaceEmbeddingConfig"] = _HuggingFaceEmbeddingConfig - return _HuggingFaceEmbeddingConfig - - if name == "OobaboogaConfig": - from .llms.oobabooga.chat.transformation import ( - OobaboogaConfig as _OobaboogaConfig, - ) - - _globals["OobaboogaConfig"] = _OobaboogaConfig - return _OobaboogaConfig - - if name == "MaritalkConfig": - from .llms.maritalk import MaritalkConfig as _MaritalkConfig - - _globals["MaritalkConfig"] = _MaritalkConfig - return _MaritalkConfig - - if name == "OpenrouterConfig": - from .llms.openrouter.chat.transformation import ( - OpenrouterConfig as _OpenrouterConfig, - ) - - _globals["OpenrouterConfig"] = _OpenrouterConfig - return _OpenrouterConfig - - if name == "DataRobotConfig": - from .llms.datarobot.chat.transformation import ( - DataRobotConfig as _DataRobotConfig, - ) - - _globals["DataRobotConfig"] = _DataRobotConfig - return _DataRobotConfig - - if name == "AnthropicConfig": - from .llms.anthropic.chat.transformation import ( - AnthropicConfig as _AnthropicConfig, - ) - - _globals["AnthropicConfig"] = _AnthropicConfig - return _AnthropicConfig - - if name == "AnthropicTextConfig": - from .llms.anthropic.completion.transformation import ( - AnthropicTextConfig as _AnthropicTextConfig, - ) - - _globals["AnthropicTextConfig"] = _AnthropicTextConfig - return _AnthropicTextConfig - - if name == "GroqSTTConfig": - from .llms.groq.stt.transformation import GroqSTTConfig as _GroqSTTConfig - - _globals["GroqSTTConfig"] = _GroqSTTConfig - return _GroqSTTConfig - - if name == "TritonConfig": - from .llms.triton.completion.transformation import TritonConfig as _TritonConfig - - _globals["TritonConfig"] = _TritonConfig - return _TritonConfig - - if name == "TritonGenerateConfig": - from .llms.triton.completion.transformation import ( - TritonGenerateConfig as _TritonGenerateConfig, - ) - - _globals["TritonGenerateConfig"] = _TritonGenerateConfig - return _TritonGenerateConfig - - if name == "TritonInferConfig": - from .llms.triton.completion.transformation import ( - TritonInferConfig as _TritonInferConfig, - ) - - _globals["TritonInferConfig"] = _TritonInferConfig - return _TritonInferConfig - - if name == "TritonEmbeddingConfig": - from .llms.triton.embedding.transformation import ( - TritonEmbeddingConfig as _TritonEmbeddingConfig, - ) - - _globals["TritonEmbeddingConfig"] = _TritonEmbeddingConfig - return _TritonEmbeddingConfig - - if name == "HuggingFaceRerankConfig": - from .llms.huggingface.rerank.transformation import ( - HuggingFaceRerankConfig as _HuggingFaceRerankConfig, - ) - - _globals["HuggingFaceRerankConfig"] = _HuggingFaceRerankConfig - return _HuggingFaceRerankConfig - - if name == "DatabricksConfig": - from .llms.databricks.chat.transformation import ( - DatabricksConfig as _DatabricksConfig, - ) - - _globals["DatabricksConfig"] = _DatabricksConfig - return _DatabricksConfig - - if name == "DatabricksEmbeddingConfig": - from .llms.databricks.embed.transformation import ( - DatabricksEmbeddingConfig as _DatabricksEmbeddingConfig, - ) - - _globals["DatabricksEmbeddingConfig"] = _DatabricksEmbeddingConfig - return _DatabricksEmbeddingConfig - - if name == "PredibaseConfig": - from .llms.predibase.chat.transformation import ( - PredibaseConfig as _PredibaseConfig, - ) - - _globals["PredibaseConfig"] = _PredibaseConfig - return _PredibaseConfig - - if name == "ReplicateConfig": - from .llms.replicate.chat.transformation import ( - ReplicateConfig as _ReplicateConfig, - ) - - _globals["ReplicateConfig"] = _ReplicateConfig - return _ReplicateConfig - - if name == "SnowflakeConfig": - from .llms.snowflake.chat.transformation import ( - SnowflakeConfig as _SnowflakeConfig, - ) - - _globals["SnowflakeConfig"] = _SnowflakeConfig - return _SnowflakeConfig - - if name == "CohereRerankConfig": - from .llms.cohere.rerank.transformation import ( - CohereRerankConfig as _CohereRerankConfig, - ) - - _globals["CohereRerankConfig"] = _CohereRerankConfig - return _CohereRerankConfig - - if name == "CohereRerankV2Config": - from .llms.cohere.rerank_v2.transformation import ( - CohereRerankV2Config as _CohereRerankV2Config, - ) - - _globals["CohereRerankV2Config"] = _CohereRerankV2Config - return _CohereRerankV2Config - - if name == "AzureAIRerankConfig": - from .llms.azure_ai.rerank.transformation import ( - AzureAIRerankConfig as _AzureAIRerankConfig, - ) - - _globals["AzureAIRerankConfig"] = _AzureAIRerankConfig - return _AzureAIRerankConfig - - if name == "InfinityRerankConfig": - from .llms.infinity.rerank.transformation import ( - InfinityRerankConfig as _InfinityRerankConfig, - ) - - _globals["InfinityRerankConfig"] = _InfinityRerankConfig - return _InfinityRerankConfig - - if name == "JinaAIRerankConfig": - from .llms.jina_ai.rerank.transformation import ( - JinaAIRerankConfig as _JinaAIRerankConfig, - ) - - _globals["JinaAIRerankConfig"] = _JinaAIRerankConfig - return _JinaAIRerankConfig - - if name == "DeepinfraRerankConfig": - from .llms.deepinfra.rerank.transformation import ( - DeepinfraRerankConfig as _DeepinfraRerankConfig, - ) - - _globals["DeepinfraRerankConfig"] = _DeepinfraRerankConfig - return _DeepinfraRerankConfig - - if name == "HostedVLLMRerankConfig": - from .llms.hosted_vllm.rerank.transformation import ( - HostedVLLMRerankConfig as _HostedVLLMRerankConfig, - ) - - _globals["HostedVLLMRerankConfig"] = _HostedVLLMRerankConfig - return _HostedVLLMRerankConfig - - if name == "NvidiaNimRerankConfig": - from .llms.nvidia_nim.rerank.transformation import ( - NvidiaNimRerankConfig as _NvidiaNimRerankConfig, - ) - - _globals["NvidiaNimRerankConfig"] = _NvidiaNimRerankConfig - return _NvidiaNimRerankConfig - - if name == "NvidiaNimRankingConfig": - from .llms.nvidia_nim.rerank.ranking_transformation import ( - NvidiaNimRankingConfig as _NvidiaNimRankingConfig, - ) - - _globals["NvidiaNimRankingConfig"] = _NvidiaNimRankingConfig - return _NvidiaNimRankingConfig - - if name == "VertexAIRerankConfig": - from .llms.vertex_ai.rerank.transformation import ( - VertexAIRerankConfig as _VertexAIRerankConfig, - ) - - _globals["VertexAIRerankConfig"] = _VertexAIRerankConfig - return _VertexAIRerankConfig - - if name == "FireworksAIRerankConfig": - from .llms.fireworks_ai.rerank.transformation import ( - FireworksAIRerankConfig as _FireworksAIRerankConfig, - ) - - _globals["FireworksAIRerankConfig"] = _FireworksAIRerankConfig - return _FireworksAIRerankConfig - - if name == "VoyageRerankConfig": - from .llms.voyage.rerank.transformation import ( - VoyageRerankConfig as _VoyageRerankConfig, - ) - - _globals["VoyageRerankConfig"] = _VoyageRerankConfig - return _VoyageRerankConfig - - if name == "ClarifaiConfig": - from .llms.clarifai.chat.transformation import ClarifaiConfig as _ClarifaiConfig - - _globals["ClarifaiConfig"] = _ClarifaiConfig - return _ClarifaiConfig - - raise AttributeError(f"LLM config lazy import: unknown attribute {name!r}") \ No newline at end of file diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py new file mode 100644 index 00000000000..698995932c1 --- /dev/null +++ b/litellm/_lazy_imports_registry.py @@ -0,0 +1,305 @@ +""" +Registry data for lazy imports. + +This module contains all the name tuples and import maps used by the lazy import system. +Separated from the handler functions for better organization. +""" + +# Cost calculator names that support lazy loading via _lazy_import_cost_calculator +COST_CALCULATOR_NAMES = ( + "completion_cost", + "cost_per_token", + "response_cost_calculator", +) + +# Litellm logging names that support lazy loading via _lazy_import_litellm_logging +LITELLM_LOGGING_NAMES = ( + "Logging", + "modify_integration", +) + +# Utils names that support lazy loading via _lazy_import_utils +UTILS_NAMES = ( + "exception_type", "get_optional_params", "get_response_string", "token_counter", + "create_pretrained_tokenizer", "create_tokenizer", "supports_function_calling", + "supports_web_search", "supports_url_context", "supports_response_schema", + "supports_parallel_function_calling", "supports_vision", "supports_audio_input", + "supports_audio_output", "supports_system_messages", "supports_reasoning", + "get_litellm_params", "acreate", "get_max_tokens", "get_model_info", + "register_prompt_template", "validate_environment", "check_valid_key", + "register_model", "encode", "decode", "_calculate_retry_after", "_should_retry", + "get_supported_openai_params", "get_api_base", "get_first_chars_messages", + "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", + "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", + "ModelResponseListIterator", "get_valid_models", +) + +# Token counter names that support lazy loading via _lazy_import_token_counter +TOKEN_COUNTER_NAMES = ( + "get_modified_max_tokens", +) + +# LLM client cache names that support lazy loading via _lazy_import_llm_client_cache +LLM_CLIENT_CACHE_NAMES = ( + "LLMClientCache", + "in_memory_llm_clients_cache", +) + +# Bedrock type names that support lazy loading via _lazy_import_bedrock_types +BEDROCK_TYPES_NAMES = ( + "COHERE_EMBEDDING_INPUT_TYPES", +) + +# Common types from litellm.types.utils that support lazy loading via +# _lazy_import_types_utils +TYPES_UTILS_NAMES = ( + "ImageObject", + "BudgetConfig", + "all_litellm_params", + "_litellm_completion_params", + "CredentialItem", + "PriorityReservationDict", + "StandardKeyGenerationConfig", + "SearchProviders", + "GenericStreamingChunk", +) + +# Caching / cache classes that support lazy loading via _lazy_import_caching +CACHING_NAMES = ( + "Cache", + "DualCache", + "RedisCache", + "InMemoryCache", +) + +# HTTP handler names that support lazy loading via _lazy_import_http_handlers +HTTP_HANDLER_NAMES = ( + "module_level_aclient", + "module_level_client", +) + +# Dotprompt integration names that support lazy loading via _lazy_import_dotprompt +DOTPROMPT_NAMES = ( + "global_prompt_manager", + "global_prompt_directory", + "set_global_prompt_directory", +) + +# LLM config classes that support lazy loading via _lazy_import_llm_configs +LLM_CONFIG_NAMES = ( + "AmazonConverseConfig", + "OpenAILikeChatConfig", + "GaladrielChatConfig", + "GithubChatConfig", + "AzureAnthropicConfig", + "BytezChatConfig", + "CompactifAIChatConfig", + "EmpowerChatConfig", + "MinimaxChatConfig", + "AiohttpOpenAIChatConfig", + "HuggingFaceChatConfig", + "HuggingFaceEmbeddingConfig", + "OobaboogaConfig", + "MaritalkConfig", + "OpenrouterConfig", + "DataRobotConfig", + "AnthropicConfig", + "AnthropicTextConfig", + "GroqSTTConfig", + "TritonConfig", + "TritonGenerateConfig", + "TritonInferConfig", + "TritonEmbeddingConfig", + "HuggingFaceRerankConfig", + "DatabricksConfig", + "DatabricksEmbeddingConfig", + "PredibaseConfig", + "ReplicateConfig", + "SnowflakeConfig", + "CohereRerankConfig", + "CohereRerankV2Config", + "AzureAIRerankConfig", + "InfinityRerankConfig", + "JinaAIRerankConfig", + "DeepinfraRerankConfig", + "HostedVLLMRerankConfig", + "NvidiaNimRerankConfig", + "NvidiaNimRankingConfig", + "VertexAIRerankConfig", + "FireworksAIRerankConfig", + "VoyageRerankConfig", + "ClarifaiConfig", + "AI21ChatConfig", +) + +# Types that support lazy loading via _lazy_import_types +TYPES_NAMES = ( + "GuardrailItem", +) + +# Import maps for registry pattern - reduces repetition +_UTILS_IMPORT_MAP = { + "exception_type": (".utils", "exception_type"), + "get_optional_params": (".utils", "get_optional_params"), + "get_response_string": (".utils", "get_response_string"), + "token_counter": (".utils", "token_counter"), + "create_pretrained_tokenizer": (".utils", "create_pretrained_tokenizer"), + "create_tokenizer": (".utils", "create_tokenizer"), + "supports_function_calling": (".utils", "supports_function_calling"), + "supports_web_search": (".utils", "supports_web_search"), + "supports_url_context": (".utils", "supports_url_context"), + "supports_response_schema": (".utils", "supports_response_schema"), + "supports_parallel_function_calling": (".utils", "supports_parallel_function_calling"), + "supports_vision": (".utils", "supports_vision"), + "supports_audio_input": (".utils", "supports_audio_input"), + "supports_audio_output": (".utils", "supports_audio_output"), + "supports_system_messages": (".utils", "supports_system_messages"), + "supports_reasoning": (".utils", "supports_reasoning"), + "get_litellm_params": (".utils", "get_litellm_params"), + "acreate": (".utils", "acreate"), + "get_max_tokens": (".utils", "get_max_tokens"), + "get_model_info": (".utils", "get_model_info"), + "register_prompt_template": (".utils", "register_prompt_template"), + "validate_environment": (".utils", "validate_environment"), + "check_valid_key": (".utils", "check_valid_key"), + "register_model": (".utils", "register_model"), + "encode": (".utils", "encode"), + "decode": (".utils", "decode"), + "_calculate_retry_after": (".utils", "_calculate_retry_after"), + "_should_retry": (".utils", "_should_retry"), + "get_supported_openai_params": (".utils", "get_supported_openai_params"), + "get_api_base": (".utils", "get_api_base"), + "get_first_chars_messages": (".utils", "get_first_chars_messages"), + "ModelResponse": (".utils", "ModelResponse"), + "ModelResponseStream": (".utils", "ModelResponseStream"), + "EmbeddingResponse": (".utils", "EmbeddingResponse"), + "ImageResponse": (".utils", "ImageResponse"), + "TranscriptionResponse": (".utils", "TranscriptionResponse"), + "TextCompletionResponse": (".utils", "TextCompletionResponse"), + "get_provider_fields": (".utils", "get_provider_fields"), + "ModelResponseListIterator": (".utils", "ModelResponseListIterator"), + "get_valid_models": (".utils", "get_valid_models"), +} + +_COST_CALCULATOR_IMPORT_MAP = { + "completion_cost": (".cost_calculator", "completion_cost"), + "cost_per_token": (".cost_calculator", "cost_per_token"), + "response_cost_calculator": (".cost_calculator", "response_cost_calculator"), +} + +_TYPES_UTILS_IMPORT_MAP = { + "ImageObject": (".types.utils", "ImageObject"), + "BudgetConfig": (".types.utils", "BudgetConfig"), + "all_litellm_params": (".types.utils", "all_litellm_params"), + "_litellm_completion_params": (".types.utils", "all_litellm_params"), # Alias + "CredentialItem": (".types.utils", "CredentialItem"), + "PriorityReservationDict": (".types.utils", "PriorityReservationDict"), + "StandardKeyGenerationConfig": (".types.utils", "StandardKeyGenerationConfig"), + "SearchProviders": (".types.utils", "SearchProviders"), + "GenericStreamingChunk": (".types.utils", "GenericStreamingChunk"), +} + +_TOKEN_COUNTER_IMPORT_MAP = { + "get_modified_max_tokens": ("litellm.litellm_core_utils.token_counter", "get_modified_max_tokens"), +} + +_BEDROCK_TYPES_IMPORT_MAP = { + "COHERE_EMBEDDING_INPUT_TYPES": ("litellm.types.llms.bedrock", "COHERE_EMBEDDING_INPUT_TYPES"), +} + +_CACHING_IMPORT_MAP = { + "Cache": ("litellm.caching.caching", "Cache"), + "DualCache": ("litellm.caching.caching", "DualCache"), + "RedisCache": ("litellm.caching.caching", "RedisCache"), + "InMemoryCache": ("litellm.caching.caching", "InMemoryCache"), +} + +_LITELLM_LOGGING_IMPORT_MAP = { + "Logging": ("litellm.litellm_core_utils.litellm_logging", "Logging"), + "modify_integration": ("litellm.litellm_core_utils.litellm_logging", "modify_integration"), +} + +_DOTPROMPT_IMPORT_MAP = { + "global_prompt_manager": ("litellm.integrations.dotprompt", "global_prompt_manager"), + "global_prompt_directory": ("litellm.integrations.dotprompt", "global_prompt_directory"), + "set_global_prompt_directory": ("litellm.integrations.dotprompt", "set_global_prompt_directory"), +} + +_TYPES_IMPORT_MAP = { + "GuardrailItem": ("litellm.types.guardrails", "GuardrailItem"), +} + +_LLM_CONFIGS_IMPORT_MAP = { + "AmazonConverseConfig": (".llms.bedrock.chat.converse_transformation", "AmazonConverseConfig"), + "OpenAILikeChatConfig": (".llms.openai_like.chat.handler", "OpenAILikeChatConfig"), + "GaladrielChatConfig": (".llms.galadriel.chat.transformation", "GaladrielChatConfig"), + "GithubChatConfig": (".llms.github.chat.transformation", "GithubChatConfig"), + "AzureAnthropicConfig": (".llms.azure_ai.anthropic.transformation", "AzureAnthropicConfig"), + "BytezChatConfig": (".llms.bytez.chat.transformation", "BytezChatConfig"), + "CompactifAIChatConfig": (".llms.compactifai.chat.transformation", "CompactifAIChatConfig"), + "EmpowerChatConfig": (".llms.empower.chat.transformation", "EmpowerChatConfig"), + "MinimaxChatConfig": (".llms.minimax.chat.transformation", "MinimaxChatConfig"), + "AiohttpOpenAIChatConfig": (".llms.aiohttp_openai.chat.transformation", "AiohttpOpenAIChatConfig"), + "HuggingFaceChatConfig": (".llms.huggingface.chat.transformation", "HuggingFaceChatConfig"), + "HuggingFaceEmbeddingConfig": (".llms.huggingface.embedding.transformation", "HuggingFaceEmbeddingConfig"), + "OobaboogaConfig": (".llms.oobabooga.chat.transformation", "OobaboogaConfig"), + "MaritalkConfig": (".llms.maritalk", "MaritalkConfig"), + "OpenrouterConfig": (".llms.openrouter.chat.transformation", "OpenrouterConfig"), + "DataRobotConfig": (".llms.datarobot.chat.transformation", "DataRobotConfig"), + "AnthropicConfig": (".llms.anthropic.chat.transformation", "AnthropicConfig"), + "AnthropicTextConfig": (".llms.anthropic.completion.transformation", "AnthropicTextConfig"), + "GroqSTTConfig": (".llms.groq.stt.transformation", "GroqSTTConfig"), + "TritonConfig": (".llms.triton.completion.transformation", "TritonConfig"), + "TritonGenerateConfig": (".llms.triton.completion.transformation", "TritonGenerateConfig"), + "TritonInferConfig": (".llms.triton.completion.transformation", "TritonInferConfig"), + "TritonEmbeddingConfig": (".llms.triton.embedding.transformation", "TritonEmbeddingConfig"), + "HuggingFaceRerankConfig": (".llms.huggingface.rerank.transformation", "HuggingFaceRerankConfig"), + "DatabricksConfig": (".llms.databricks.chat.transformation", "DatabricksConfig"), + "DatabricksEmbeddingConfig": (".llms.databricks.embed.transformation", "DatabricksEmbeddingConfig"), + "PredibaseConfig": (".llms.predibase.chat.transformation", "PredibaseConfig"), + "ReplicateConfig": (".llms.replicate.chat.transformation", "ReplicateConfig"), + "SnowflakeConfig": (".llms.snowflake.chat.transformation", "SnowflakeConfig"), + "CohereRerankConfig": (".llms.cohere.rerank.transformation", "CohereRerankConfig"), + "CohereRerankV2Config": (".llms.cohere.rerank_v2.transformation", "CohereRerankV2Config"), + "AzureAIRerankConfig": (".llms.azure_ai.rerank.transformation", "AzureAIRerankConfig"), + "InfinityRerankConfig": (".llms.infinity.rerank.transformation", "InfinityRerankConfig"), + "JinaAIRerankConfig": (".llms.jina_ai.rerank.transformation", "JinaAIRerankConfig"), + "DeepinfraRerankConfig": (".llms.deepinfra.rerank.transformation", "DeepinfraRerankConfig"), + "HostedVLLMRerankConfig": (".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig"), + "NvidiaNimRerankConfig": (".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig"), + "NvidiaNimRankingConfig": (".llms.nvidia_nim.rerank.ranking_transformation", "NvidiaNimRankingConfig"), + "VertexAIRerankConfig": (".llms.vertex_ai.rerank.transformation", "VertexAIRerankConfig"), + "FireworksAIRerankConfig": (".llms.fireworks_ai.rerank.transformation", "FireworksAIRerankConfig"), + "VoyageRerankConfig": (".llms.voyage.rerank.transformation", "VoyageRerankConfig"), + "ClarifaiConfig": (".llms.clarifai.chat.transformation", "ClarifaiConfig"), + "AI21ChatConfig": (".llms.ai21.chat.transformation", "AI21ChatConfig"), +} + +# Export all name tuples and import maps for use in _lazy_imports.py +__all__ = [ + # Name tuples + "COST_CALCULATOR_NAMES", + "LITELLM_LOGGING_NAMES", + "UTILS_NAMES", + "TOKEN_COUNTER_NAMES", + "LLM_CLIENT_CACHE_NAMES", + "BEDROCK_TYPES_NAMES", + "TYPES_UTILS_NAMES", + "CACHING_NAMES", + "HTTP_HANDLER_NAMES", + "DOTPROMPT_NAMES", + "LLM_CONFIG_NAMES", + "TYPES_NAMES", + # Import maps + "_UTILS_IMPORT_MAP", + "_COST_CALCULATOR_IMPORT_MAP", + "_TYPES_UTILS_IMPORT_MAP", + "_TOKEN_COUNTER_IMPORT_MAP", + "_BEDROCK_TYPES_IMPORT_MAP", + "_CACHING_IMPORT_MAP", + "_LITELLM_LOGGING_IMPORT_MAP", + "_DOTPROMPT_IMPORT_MAP", + "_TYPES_IMPORT_MAP", + "_LLM_CONFIGS_IMPORT_MAP", +] + From ed4a4c13d64eb281df7297152416b0b2355b5bb7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 11:46:35 -0800 Subject: [PATCH 043/388] Base commit --- tests/test_litellm/proxy/test_proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index fd7036b9940..dadb3281192 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -525,7 +525,7 @@ mock_prisma = MockPrisma() @pytest.mark.asyncio async def test_aaaproxy_startup_master_key(mock_prisma, monkeypatch, tmp_path): """ - Test that master_key is correctly loaded from either config.yaml or environment variables + Test that master_key is correctly loaded from either config.yaml or environment variables. """ import yaml from fastapi import FastAPI From fccd2d1e87ae1a2a5762eface9f6a5eaec871953 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 11:45:31 -0800 Subject: [PATCH 044/388] Fix UI disappearing for development instances --- docker/Dockerfile.non_root | 24 ++--- litellm/proxy/proxy_server.py | 59 ++++++------- tests/test_litellm/proxy/test_proxy_server.py | 88 +++++++++++++++---- 3 files changed, 110 insertions(+), 61 deletions(-) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 7e9147a124e..af1bb5b2022 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -40,7 +40,7 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI using the upstream command order while keeping a single RUN layer -RUN mkdir -p /tmp/litellm_ui && \ +RUN mkdir -p /var/lib/litellm/ui && \ npm install -g npm@latest && npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ @@ -49,10 +49,10 @@ RUN mkdir -p /tmp/litellm_ui && \ rm -f package-lock.json && \ npm install --legacy-peer-deps && \ npm run build && \ - cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \ - mkdir -p /tmp/litellm_assets && \ - cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \ - ( cd /tmp/litellm_ui && \ + cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ + mkdir -p /var/lib/litellm/assets && \ + cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ + ( cd /var/lib/litellm/ui && \ for html_file in *.html; do \ if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ folder_name="${html_file%.html}" && \ @@ -111,8 +111,8 @@ COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /ap COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf COPY --from=builder /app/schema.prisma /app/ COPY --from=builder /wheels/ /wheels/ -COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui -COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets +COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui +COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets COPY --from=builder /app/.cache /app/.cache COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras COPY --from=builder \ @@ -145,8 +145,8 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ # Permissions, cleanup, and Prisma prep RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ - mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \ - chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \ + mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \ + chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \ pip uninstall jwt -y || true && \ pip uninstall PyJWT -y || true && \ pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \ @@ -156,11 +156,11 @@ RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \ LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \ chmod -R g+rX $PRISMA_PATH && \ chmod -R g+rX /app/.cache && \ diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f754e52796f..a1be8153b29 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -946,20 +946,19 @@ try: # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) # and ensures extensionless routes like /ui/login work via /index.html. is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - runtime_ui_path = "/tmp/litellm_ui" - if _dir_has_content(runtime_ui_path): - if is_non_root: + # Only use runtime UI path in Docker/non-root environments + # In local development, use the packaged UI directly + if is_non_root: + # Use /var/lib/litellm/ui for Docker (more secure than /tmp) + runtime_ui_path = "/var/lib/litellm/ui" + + if _dir_has_content(runtime_ui_path): verbose_proxy_logger.info( f"Using pre-built UI for non-root Docker: {runtime_ui_path}" ) + ui_path = runtime_ui_path else: - verbose_proxy_logger.info( - f"Using cached runtime UI directory: {runtime_ui_path}" - ) - ui_path = runtime_ui_path - else: - if is_non_root: verbose_proxy_logger.error( f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." ) @@ -967,32 +966,32 @@ try: f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" ) - try: - os.makedirs(runtime_ui_path, exist_ok=True) - if not _dir_has_content(runtime_ui_path) and _dir_has_content( - packaged_ui_path - ): - shutil.copytree( - packaged_ui_path, - runtime_ui_path, - dirs_exist_ok=True, - ) - except Exception as e: - if is_non_root: + try: + os.makedirs(runtime_ui_path, exist_ok=True) + if not _dir_has_content(runtime_ui_path) and _dir_has_content( + packaged_ui_path + ): + shutil.copytree( + packaged_ui_path, + runtime_ui_path, + dirs_exist_ok=True, + ) + except Exception as e: verbose_proxy_logger.exception( f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" ) - else: - if _dir_has_content(runtime_ui_path): - if is_non_root: + else: + if _dir_has_content(runtime_ui_path): verbose_proxy_logger.info( f"Using populated UI for non-root Docker: {runtime_ui_path}" ) - else: - verbose_proxy_logger.info( - f"Using populated runtime UI directory: {runtime_ui_path}" - ) - ui_path = runtime_ui_path + ui_path = runtime_ui_path + else: + # Local development: use packaged UI directly, no runtime copy needed + verbose_proxy_logger.info( + f"Using packaged UI directory for local development: {packaged_ui_path}" + ) + ui_path = packaged_ui_path # Only modify files if a custom server root path is set if server_root_path and server_root_path != "/": @@ -8885,7 +8884,7 @@ def get_image(): default_site_logo = os.path.join(current_dir, "logo.jpg") is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - assets_dir = "/tmp/litellm_assets" if is_non_root else current_dir + assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir if is_non_root: os.makedirs(assets_dir, exist_ok=True) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index dadb3281192..737877b0ed2 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -273,6 +273,10 @@ def test_sso_key_generate_shows_deprecation_banner(client_no_auth, monkeypatch): def test_restructure_ui_html_files_handles_nested_routes(tmp_path): + """ + Test that _restructure_ui_html_files correctly restructures HTML files. + Note: This function is only called when is_non_root is True (ui_path != packaged_ui_path). + """ from litellm.proxy import proxy_server ui_root = tmp_path / "ui" @@ -306,7 +310,10 @@ def test_restructure_ui_html_files_handles_nested_routes(tmp_path): def test_ui_extensionless_route_requires_restructure(tmp_path): - """Regression for non-root fallback: /ui/login expects login/index.html.""" + """ + Regression for non-root fallback: /ui/login expects login/index.html. + Note: Restructuring only happens when is_non_root is True (ui_path != packaged_ui_path). + """ from litellm.proxy import proxy_server @@ -331,6 +338,49 @@ def test_ui_extensionless_route_requires_restructure(tmp_path): assert "login" in response.text +def test_restructure_only_happens_when_non_root(monkeypatch): + """ + Test that restructuring logic only executes when LITELLM_NON_ROOT is true. + When is_non_root is False, ui_path == packaged_ui_path, so restructuring is skipped. + """ + # Test Case 1: is_non_root is True - ui_path != packaged_ui_path, so restructuring should happen + monkeypatch.setenv("LITELLM_NON_ROOT", "true") + + runtime_ui_path = "/var/lib/litellm/ui" + packaged_ui_path = "/some/packaged/ui/path" + + # Simulate the logic from proxy_server.py + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + if is_non_root: + ui_path = runtime_ui_path + else: + ui_path = packaged_ui_path + + # This is the condition that determines if restructuring happens + should_restructure = ui_path != packaged_ui_path + + assert is_non_root is True + assert should_restructure is True + assert ui_path == runtime_ui_path + + # Test Case 2: is_non_root is False - ui_path == packaged_ui_path, so restructuring should NOT happen + monkeypatch.delenv("LITELLM_NON_ROOT", raising=False) + + # Simulate the logic from proxy_server.py + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + if is_non_root: + ui_path = runtime_ui_path + else: + ui_path = packaged_ui_path + + # This is the condition that determines if restructuring happens + should_restructure = ui_path != packaged_ui_path + + assert is_non_root is False + assert should_restructure is False + assert ui_path == packaged_ui_path + + @pytest.mark.asyncio async def test_initialize_scheduled_jobs_credentials(monkeypatch): """ @@ -2856,9 +2906,9 @@ def test_root_redirect_when_docs_url_not_root_and_redirect_url_set(monkeypatch): assert response.headers["location"] == test_redirect_url -def test_get_image_non_root_uses_tmp_assets_dir(monkeypatch): +def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): """ - Test that get_image uses /tmp/litellm_assets when LITELLM_NON_ROOT is true. + Test that get_image uses /var/lib/litellm/assets when LITELLM_NON_ROOT is true. """ from unittest.mock import patch @@ -2887,14 +2937,14 @@ def test_get_image_non_root_uses_tmp_assets_dir(monkeypatch): # Call the function get_image() - # Verify makedirs was called with /tmp/litellm_assets - mock_makedirs.assert_called_once_with("/tmp/litellm_assets", exist_ok=True) + # Verify makedirs was called with /var/lib/litellm/assets + mock_makedirs.assert_called_once_with("/var/lib/litellm/assets", exist_ok=True) def test_get_image_non_root_fallback_to_default_logo(monkeypatch): """ Test that get_image falls back to default_site_logo when logo doesn't exist - in /tmp/litellm_assets for non-root case. + in /var/lib/litellm/assets for non-root case. """ from unittest.mock import patch @@ -2904,13 +2954,13 @@ def test_get_image_non_root_fallback_to_default_logo(monkeypatch): monkeypatch.setenv("LITELLM_NON_ROOT", "true") monkeypatch.delenv("UI_LOGO_PATH", raising=False) - # Track path.exists calls to verify it checks /tmp/litellm_assets/logo.jpg + # Track path.exists calls to verify it checks /var/lib/litellm/assets/logo.jpg exists_calls = [] def exists_side_effect(path): exists_calls.append(path) - # Return False for /tmp/litellm_assets/logo.jpg to trigger fallback - if "/tmp/litellm_assets/logo.jpg" in path: + # Return False for /var/lib/litellm/assets/logo.jpg to trigger fallback + if "/var/lib/litellm/assets/logo.jpg" in path: return False return True @@ -2933,13 +2983,13 @@ def test_get_image_non_root_fallback_to_default_logo(monkeypatch): # Call the function get_image() - # Verify makedirs was called with /tmp/litellm_assets - mock_makedirs.assert_called_once_with("/tmp/litellm_assets", exist_ok=True) + # Verify makedirs was called with /var/lib/litellm/assets + mock_makedirs.assert_called_once_with("/var/lib/litellm/assets", exist_ok=True) - # Verify that exists was called to check /tmp/litellm_assets/logo.jpg - tmp_logo_path = "/tmp/litellm_assets/logo.jpg" - assert any(tmp_logo_path in str(call) for call in exists_calls), \ - f"Should check if {tmp_logo_path} exists" + # Verify that exists was called to check /var/lib/litellm/assets/logo.jpg + assets_logo_path = "/var/lib/litellm/assets/logo.jpg" + assert any(assets_logo_path in str(call) for call in exists_calls), \ + f"Should check if {assets_logo_path} exists" # Verify FileResponse was called (with fallback logo) assert mock_file_response.called, "FileResponse should be called" @@ -2976,12 +3026,12 @@ def test_get_image_root_case_uses_current_dir(monkeypatch): # Call the function get_image() - # Verify makedirs was NOT called with /tmp/litellm_assets (should not create it for root case) - tmp_assets_calls = [ + # Verify makedirs was NOT called with /var/lib/litellm/assets (should not create it for root case) + var_lib_assets_calls = [ call for call in mock_makedirs.call_args_list - if "/tmp/litellm_assets" in str(call) + if "/var/lib/litellm/assets" in str(call) ] - assert len(tmp_assets_calls) == 0, "Should not create /tmp/litellm_assets for root case" + assert len(var_lib_assets_calls) == 0, "Should not create /var/lib/litellm/assets for root case" # Verify FileResponse was called assert mock_file_response.called, "FileResponse should be called" From 449a5868890200c1341e15c09ed489c786658c8f Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Tue, 23 Dec 2025 12:31:31 -0800 Subject: [PATCH 045/388] [Refactor]: Complete lazy loading migration for all 180+ LLM config classes (#18392) --- litellm/__init__.py | 423 ++++++++---------- litellm/_lazy_imports_registry.py | 280 ++++++++++++ .../bedrock/image_generation/image_handler.py | 5 +- 3 files changed, 479 insertions(+), 229 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 9da367c3672..ecb101e2de5 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1066,32 +1066,11 @@ from .utils import client from .llms.custom_llm import CustomLLM from .llms.anthropic.common_utils import AnthropicModelInfo from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config -from .llms.meta_llama.chat.transformation import LlamaAPIConfig -from .llms.anthropic.experimental_pass_through.messages.transformation import ( - AnthropicMessagesConfig, -) -from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( - AmazonAnthropicClaudeMessagesConfig, -) -from .llms.together_ai.chat import TogetherAIConfig -from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig -from .llms.cloudflare.chat.transformation import CloudflareChatConfig -from .llms.novita.chat.transformation import NovitaConfig from .llms.deprecated_providers.palm import ( PalmConfig, ) # here to prevent breaking changes -from .llms.nlp_cloud.chat.handler import NLPCloudConfig -from .llms.petals.completion.transformation import PetalsConfig from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig -from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - VertexGeminiConfig as VertexAIConfig, -) from .llms.gemini.common_utils import GeminiModelInfo -from .llms.gemini.chat.transformation import ( - GoogleAIStudioGeminiConfig, - GoogleAIStudioGeminiConfig as GeminiConfig, # aliased to maintain backwards compatibility -) from .llms.vertex_ai.vertex_embeddings.transformation import ( @@ -1100,227 +1079,23 @@ from .llms.vertex_ai.vertex_embeddings.transformation import ( vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig() -from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( - VertexAIAnthropicConfig, -) -from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import ( - VertexAILlama3Config, -) -from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import ( - VertexAIAi21Config, -) -from .llms.ollama.chat.transformation import OllamaChatConfig -from .llms.ollama.completion.transformation import OllamaConfig -from .llms.sagemaker.completion.transformation import SagemakerConfig -from .llms.sagemaker.chat.transformation import SagemakerChatConfig from .llms.bedrock.chat.invoke_handler import ( - AmazonCohereChatConfig, bedrock_tool_name_mappings, ) -from .llms.bedrock.common_utils import ( - AmazonBedrockGlobalConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import ( - AmazonAI21Config, -) -from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import ( - AmazonInvokeNovaConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import ( - AmazonQwen2Config, -) -from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import ( - AmazonQwen3Config, -) -from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import ( - AmazonAnthropicConfig, -) -from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( - AmazonAnthropicClaudeConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import ( - AmazonCohereConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import ( - AmazonLlamaConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import ( - AmazonDeepSeekR1Config, -) -from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import ( - AmazonMistralConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import ( - AmazonTitanConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import ( - AmazonTwelveLabsPegasusConfig, -) -from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( - AmazonInvokeConfig, -) -from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( - AmazonBedrockOpenAIConfig, -) - -from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig -from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config -from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig -from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config -from .llms.bedrock.embed.amazon_titan_multimodal_transformation import ( - AmazonTitanMultimodalEmbeddingG1Config, -) from .llms.bedrock.embed.amazon_titan_v2_transformation import ( AmazonTitanV2Config, ) -from .llms.cohere.chat.transformation import CohereChatConfig -from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig -from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig -from .llms.bedrock.embed.twelvelabs_marengo_transformation import ( - TwelveLabsMarengoEmbeddingConfig, -) -from .llms.bedrock.embed.amazon_nova_transformation import ( - AmazonNovaEmbeddingConfig, -) -from .llms.openai.openai import OpenAIConfig, MistralEmbeddingConfig -from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig -from .llms.deepinfra.chat.transformation import DeepInfraConfig -from .llms.deepgram.audio_transcription.transformation import ( - DeepgramAudioTranscriptionConfig, -) from .llms.topaz.common_utils import TopazModelInfo -from .llms.topaz.image_variations.transformation import TopazImageVariationConfig -from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig -from .llms.groq.chat.transformation import GroqChatConfig -from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig -from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig -from .llms.voyage.embedding.transformation_contextual import ( - VoyageContextualEmbeddingConfig, -) -from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig -from .llms.azure_ai.chat.transformation import AzureAIStudioConfig -from .llms.mistral.chat.transformation import MistralConfig -from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig -from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig -from .llms.azure.responses.o_series_transformation import ( - AzureOpenAIOSeriesResponsesAPIConfig, -) -from .llms.xai.responses.transformation import XAIResponsesAPIConfig -from .llms.litellm_proxy.responses.transformation import ( - LiteLLMProxyResponsesAPIConfig, -) -from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig -from .llms.openai.chat.o_series_transformation import ( - OpenAIOSeriesConfig as OpenAIO1Config, # maintain backwards compatibility - OpenAIOSeriesConfig, -) -from .llms.anthropic.skills.transformation import AnthropicSkillsConfig -from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig -from .llms.gradient_ai.chat.transformation import GradientAIConfig - -openaiOSeriesConfig = OpenAIOSeriesConfig() -from .llms.openai.chat.gpt_transformation import ( - OpenAIGPTConfig, -) -from .llms.openai.chat.gpt_5_transformation import ( - OpenAIGPT5Config, -) -from .llms.openai.transcriptions.whisper_transformation import ( - OpenAIWhisperAudioTranscriptionConfig, -) -from .llms.openai.transcriptions.gpt_transformation import ( - OpenAIGPTAudioTranscriptionConfig, -) - -openAIGPTConfig = OpenAIGPTConfig() -from .llms.openai.chat.gpt_audio_transformation import ( - OpenAIGPTAudioConfig, -) - -openAIGPTAudioConfig = OpenAIGPTAudioConfig() -openAIGPT5Config = OpenAIGPT5Config() - -from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig -from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig - -nvidiaNimConfig = NvidiaNimConfig() -nvidiaNimEmbeddingConfig = NvidiaNimEmbeddingConfig() - -from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig -from .llms.cerebras.chat import CerebrasConfig -from .llms.baseten.chat import BasetenConfig -from .llms.sambanova.chat import SambanovaConfig -from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig -from .llms.fireworks_ai.chat.transformation import FireworksAIConfig -from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig -from .llms.fireworks_ai.audio_transcription.transformation import ( - FireworksAIAudioTranscriptionConfig, -) -from .llms.fireworks_ai.embed.fireworks_ai_transformation import ( - FireworksAIEmbeddingConfig, -) -from .llms.friendliai.chat.transformation import FriendliaiChatConfig -from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig -from .llms.xai.chat.transformation import XAIChatConfig +# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access +# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo -from .llms.zai.chat.transformation import ZAIChatConfig -from .llms.aiml.chat.transformation import AIMLChatConfig -from .llms.volcengine.chat.transformation import ( - VolcEngineChatConfig as VolcEngineConfig, -) -from .llms.codestral.completion.transformation import CodestralTextCompletionConfig from .llms.azure.azure import ( AzureOpenAIError, - AzureOpenAIAssistantsAPIConfig, ) -from .llms.heroku.chat.transformation import HerokuChatConfig -from .llms.cometapi.chat.transformation import CometAPIConfig -from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig -from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config -from .llms.azure.completion.transformation import AzureOpenAITextConfig -from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig -from .llms.llamafile.chat.transformation import LlamafileChatConfig -from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig -from .llms.vllm.completion.transformation import VLLMConfig -from .llms.deepseek.chat.transformation import DeepSeekChatConfig -from .llms.lm_studio.chat.transformation import LMStudioChatConfig -from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig -from .llms.nscale.chat.transformation import NscaleConfig -from .llms.perplexity.chat.transformation import PerplexityChatConfig -from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config -from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig -from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig -from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig -from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig -from .llms.watsonx.audio_transcription.transformation import ( - IBMWatsonXAudioTranscriptionConfig, -) -from .llms.github_copilot.chat.transformation import GithubCopilotConfig -from .llms.github_copilot.responses.transformation import ( - GithubCopilotResponsesAPIConfig, -) -from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig -from .llms.nebius.chat.transformation import NebiusConfig -from .llms.wandb.chat.transformation import WandbConfig -from .llms.dashscope.chat.transformation import DashScopeChatConfig -from .llms.moonshot.chat.transformation import MoonshotChatConfig # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) -from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig -from .llms.v0.chat.transformation import V0ChatConfig -from .llms.oci.chat.transformation import OCIChatConfig -from .llms.morph.chat.transformation import MorphChatConfig -from .llms.ragflow.chat.transformation import RAGFlowConfig -from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig -from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig -from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig -from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig -from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig -from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig -from .llms.lemonade.chat.transformation import LemonadeChatConfig -from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig -from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig +# All remaining configs are now lazy loaded - see _lazy_imports_registry.py ## Lazy loading this is not straightforward, will leave it here for now. from .main import * # type: ignore @@ -1517,6 +1292,167 @@ if TYPE_CHECKING: from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig + from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig + from .llms.together_ai.completion.transformation import TogetherAITextCompletionConfig as TogetherAITextCompletionConfig + from .llms.cloudflare.chat.transformation import CloudflareChatConfig as CloudflareChatConfig + from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig + from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig + from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig + from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig + from .llms.sagemaker.completion.transformation import SagemakerConfig as SagemakerConfig + from .llms.sagemaker.chat.transformation import SagemakerChatConfig as SagemakerChatConfig + from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig + from .llms.anthropic.experimental_pass_through.messages.transformation import AnthropicMessagesConfig as AnthropicMessagesConfig + from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig + from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig + from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as VertexGeminiConfig + from .llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig + from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig as VertexAIAnthropicConfig + from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import VertexAILlama3Config as VertexAILlama3Config + from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import VertexAIAi21Config as VertexAIAi21Config + from .llms.bedrock.chat.invoke_handler import AmazonCohereChatConfig as AmazonCohereChatConfig + from .llms.bedrock.common_utils import AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig + from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import AmazonAI21Config as AmazonAI21Config + from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import AmazonInvokeNovaConfig as AmazonInvokeNovaConfig + from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import AmazonQwen2Config as AmazonQwen2Config + from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import AmazonQwen3Config as AmazonQwen3Config + from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import AmazonAnthropicConfig as AmazonAnthropicConfig + from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig + from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import AmazonCohereConfig as AmazonCohereConfig + from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import AmazonLlamaConfig as AmazonLlamaConfig + from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import AmazonDeepSeekR1Config as AmazonDeepSeekR1Config + from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import AmazonMistralConfig as AmazonMistralConfig + from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import AmazonTitanConfig as AmazonTitanConfig + from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig + from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import AmazonInvokeConfig as AmazonInvokeConfig + from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig + from .llms.bedrock.image_generation.amazon_stability1_transformation import AmazonStabilityConfig as AmazonStabilityConfig + from .llms.bedrock.image_generation.amazon_stability3_transformation import AmazonStability3Config as AmazonStability3Config + from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import AmazonNovaCanvasConfig as AmazonNovaCanvasConfig + from .llms.bedrock.embed.amazon_titan_g1_transformation import AmazonTitanG1Config as AmazonTitanG1Config + from .llms.bedrock.embed.amazon_titan_multimodal_transformation import AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config + from .llms.cohere.chat.v2_transformation import CohereV2ChatConfig as CohereV2ChatConfig + from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig + from .llms.bedrock.embed.twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig + from .llms.bedrock.embed.amazon_nova_transformation import AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig + from .llms.openai.openai import OpenAIConfig as OpenAIConfig, MistralEmbeddingConfig as MistralEmbeddingConfig + from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig as OpenAIImageVariationConfig + from .llms.deepgram.audio_transcription.transformation import DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig + from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig + from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig + from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig + from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig + from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig + from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig as InfinityEmbeddingConfig + from .llms.azure_ai.chat.transformation import AzureAIStudioConfig as AzureAIStudioConfig + from .llms.mistral.chat.transformation import MistralConfig as MistralConfig + from .llms.openai.responses.transformation import OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig + from .llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig + from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig + from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig + from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig + from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig + from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config + from .llms.anthropic.skills.transformation import AnthropicSkillsConfig as AnthropicSkillsConfig + from .llms.base_llm.skills.transformation import BaseSkillsAPIConfig as BaseSkillsAPIConfig + from .llms.gradient_ai.chat.transformation import GradientAIConfig as GradientAIConfig + from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig + from .llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config as OpenAIGPT5Config + from .llms.openai.transcriptions.whisper_transformation import OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig + from .llms.openai.transcriptions.gpt_transformation import OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig + from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig + from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig + from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig + + # Type stubs for lazy-loaded config instances + openaiOSeriesConfig: OpenAIOSeriesConfig + openAIGPTConfig: OpenAIGPTConfig + openAIGPTAudioConfig: OpenAIGPTAudioConfig + openAIGPT5Config: OpenAIGPT5Config + nvidiaNimConfig: NvidiaNimConfig + nvidiaNimEmbeddingConfig: NvidiaNimEmbeddingConfig + + # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference + from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig + from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig + from .llms.sap.chat.transformation import GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig + from .llms.sap.embed.transformation import GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig + from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config as _AzureOpenAIO1Config + from .llms.perplexity.chat.transformation import PerplexityChatConfig as _PerplexityChatConfig + from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig + from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig as _IBMWatsonXChatConfig + from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig as _IBMWatsonXAIConfig + from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig + from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig + from .llms.llamafile.chat.transformation import LlamafileChatConfig as _LlamafileChatConfig + from .llms.lm_studio.chat.transformation import LMStudioChatConfig as _LMStudioChatConfig + from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig + from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig + from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig + + # Type stubs for lazy-loaded config classes (to help mypy understand types) + VLLMConfig: Type[_VLLMConfig] + DeepSeekChatConfig: Type[_DeepSeekChatConfig] + GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig] + GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig] + AzureOpenAIO1Config: Type[_AzureOpenAIO1Config] + PerplexityChatConfig: Type[_PerplexityChatConfig] + NscaleConfig: Type[_NscaleConfig] + IBMWatsonXChatConfig: Type[_IBMWatsonXChatConfig] + IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig] + LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig] + DeepInfraConfig: Type[_DeepInfraConfig] + LlamafileChatConfig: Type[_LlamafileChatConfig] + LMStudioChatConfig: Type[_LMStudioChatConfig] + LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig] + IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig] + VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig + + from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig + from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig + from .llms.baseten.chat import BasetenConfig as BasetenConfig + from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig + from .llms.sambanova.embedding.transformation import SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig + from .llms.fireworks_ai.chat.transformation import FireworksAIConfig as FireworksAIConfig + from .llms.fireworks_ai.completion.transformation import FireworksAITextCompletionConfig as FireworksAITextCompletionConfig + from .llms.fireworks_ai.audio_transcription.transformation import FireworksAIAudioTranscriptionConfig as FireworksAIAudioTranscriptionConfig + from .llms.fireworks_ai.embed.fireworks_ai_transformation import FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig + from .llms.friendliai.chat.transformation import FriendliaiChatConfig as FriendliaiChatConfig + from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig as JinaAIEmbeddingConfig + from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig + from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig + from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig + from .llms.volcengine.chat.transformation import VolcEngineChatConfig as VolcEngineChatConfig, VolcEngineChatConfig as VolcEngineConfig + from .llms.codestral.completion.transformation import CodestralTextCompletionConfig as CodestralTextCompletionConfig + from .llms.azure.azure import AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig + from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig + from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig + from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig as AzureOpenAIConfig + from .llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config as AzureOpenAIGPT5Config + from .llms.azure.completion.transformation import AzureOpenAITextConfig as AzureOpenAITextConfig + from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig as HostedVLLMChatConfig + from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig + from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig + from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig + from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig + from .llms.wandb.chat.transformation import WandbConfig as WandbConfig + from .llms.dashscope.chat.transformation import DashScopeChatConfig as DashScopeChatConfig + from .llms.moonshot.chat.transformation import MoonshotChatConfig as MoonshotChatConfig + from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatConfig as DockerModelRunnerChatConfig + from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig + from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig + from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig + from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig + from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig as LambdaAIChatConfig + from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig as HyperbolicChatConfig + from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig as VercelAIGatewayConfig + from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig as OVHCloudChatConfig + from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig + from .llms.cometapi.embed.transformation import CometAPIEmbeddingConfig as CometAPIEmbeddingConfig + from .llms.lemonade.chat.transformation import LemonadeChatConfig as LemonadeChatConfig + from .llms.snowflake.embedding.transformation import SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig + from .llms.amazon_nova.chat.transformation import AmazonNovaChatConfig as AmazonNovaChatConfig from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( @@ -1594,6 +1530,37 @@ def __getattr__(name: str) -> Any: from .main import encoding as _encoding _globals["encoding"] = _encoding return _globals["encoding"] + + # Lazy load openaiOSeriesConfig instance + if name == "openaiOSeriesConfig": + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + if "openaiOSeriesConfig" not in _globals: + # Import the config class and instantiate it + config_class = __getattr__("OpenAIOSeriesConfig") + _globals["openaiOSeriesConfig"] = config_class() + return _globals["openaiOSeriesConfig"] + + # Lazy load other config instances + _config_instances = { + "openAIGPTConfig": "OpenAIGPTConfig", + "openAIGPTAudioConfig": "OpenAIGPTAudioConfig", + "openAIGPT5Config": "OpenAIGPT5Config", + "nvidiaNimConfig": "NvidiaNimConfig", + "nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig", + } + if name in _config_instances: + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + if name not in _globals: + # Import the config class and instantiate it + config_class = __getattr__(_config_instances[name]) + _globals[name] = config_class() + return _globals[name] + + # Handle OpenAIO1Config alias + if name == "OpenAIO1Config": + return __getattr__("OpenAIOSeriesConfig") raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 698995932c1..31ef0cf69ff 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -130,6 +130,146 @@ LLM_CONFIG_NAMES = ( "VoyageRerankConfig", "ClarifaiConfig", "AI21ChatConfig", + "LlamaAPIConfig", + "TogetherAITextCompletionConfig", + "CloudflareChatConfig", + "NovitaConfig", + "PetalsConfig", + "OllamaChatConfig", + "OllamaConfig", + "SagemakerConfig", + "SagemakerChatConfig", + "CohereChatConfig", + "AnthropicMessagesConfig", + "AmazonAnthropicClaudeMessagesConfig", + "TogetherAIConfig", + "NLPCloudConfig", + "VertexGeminiConfig", + "GoogleAIStudioGeminiConfig", + "VertexAIAnthropicConfig", + "VertexAILlama3Config", + "VertexAIAi21Config", + "AmazonCohereChatConfig", + "AmazonBedrockGlobalConfig", + "AmazonAI21Config", + "AmazonInvokeNovaConfig", + "AmazonQwen2Config", + "AmazonQwen3Config", + # Aliases for backwards compatibility + "VertexAIConfig", # Alias for VertexGeminiConfig + "GeminiConfig", # Alias for GoogleAIStudioGeminiConfig + "AmazonAnthropicConfig", + "AmazonAnthropicClaudeConfig", + "AmazonCohereConfig", + "AmazonLlamaConfig", + "AmazonDeepSeekR1Config", + "AmazonMistralConfig", + "AmazonTitanConfig", + "AmazonTwelveLabsPegasusConfig", + "AmazonInvokeConfig", + "AmazonBedrockOpenAIConfig", + "AmazonStabilityConfig", + "AmazonStability3Config", + "AmazonNovaCanvasConfig", + "AmazonTitanG1Config", + "AmazonTitanMultimodalEmbeddingG1Config", + "CohereV2ChatConfig", + "BedrockCohereEmbeddingConfig", + "TwelveLabsMarengoEmbeddingConfig", + "AmazonNovaEmbeddingConfig", + "OpenAIConfig", + "MistralEmbeddingConfig", + "OpenAIImageVariationConfig", + "DeepInfraConfig", + "DeepgramAudioTranscriptionConfig", + "TopazImageVariationConfig", + "OpenAITextCompletionConfig", + "GroqChatConfig", + "GenAIHubOrchestrationConfig", + "VoyageEmbeddingConfig", + "VoyageContextualEmbeddingConfig", + "InfinityEmbeddingConfig", + "AzureAIStudioConfig", + "MistralConfig", + "OpenAIResponsesAPIConfig", + "AzureOpenAIResponsesAPIConfig", + "AzureOpenAIOSeriesResponsesAPIConfig", + "XAIResponsesAPIConfig", + "LiteLLMProxyResponsesAPIConfig", + "GoogleAIStudioInteractionsConfig", + "OpenAIOSeriesConfig", + "AnthropicSkillsConfig", + "BaseSkillsAPIConfig", + "GradientAIConfig", + # Alias for backwards compatibility + "OpenAIO1Config", # Alias for OpenAIOSeriesConfig + "OpenAIGPTConfig", + "OpenAIGPT5Config", + "OpenAIWhisperAudioTranscriptionConfig", + "OpenAIGPTAudioTranscriptionConfig", + "OpenAIGPTAudioConfig", + "NvidiaNimConfig", + "NvidiaNimEmbeddingConfig", + "FeatherlessAIConfig", + "CerebrasConfig", + "BasetenConfig", + "SambanovaConfig", + "SambaNovaEmbeddingConfig", + "FireworksAIConfig", + "FireworksAITextCompletionConfig", + "FireworksAIAudioTranscriptionConfig", + "FireworksAIEmbeddingConfig", + "FriendliaiChatConfig", + "JinaAIEmbeddingConfig", + "XAIChatConfig", + "ZAIChatConfig", + "AIMLChatConfig", + "VolcEngineChatConfig", + "CodestralTextCompletionConfig", + "AzureOpenAIAssistantsAPIConfig", + "HerokuChatConfig", + "CometAPIConfig", + "AzureOpenAIConfig", + "AzureOpenAIGPT5Config", + "AzureOpenAITextConfig", + "HostedVLLMChatConfig", + # Alias for backwards compatibility + "VolcEngineConfig", # Alias for VolcEngineChatConfig + "LlamafileChatConfig", + "LiteLLMProxyChatConfig", + "VLLMConfig", + "DeepSeekChatConfig", + "LMStudioChatConfig", + "LmStudioEmbeddingConfig", + "NscaleConfig", + "PerplexityChatConfig", + "AzureOpenAIO1Config", + "IBMWatsonXAIConfig", + "IBMWatsonXChatConfig", + "IBMWatsonXEmbeddingConfig", + "GenAIHubEmbeddingConfig", + "IBMWatsonXAudioTranscriptionConfig", + "GithubCopilotConfig", + "GithubCopilotResponsesAPIConfig", + "GithubCopilotEmbeddingConfig", + "NebiusConfig", + "WandbConfig", + "DashScopeChatConfig", + "MoonshotChatConfig", + "DockerModelRunnerChatConfig", + "V0ChatConfig", + "OCIChatConfig", + "MorphChatConfig", + "RAGFlowConfig", + "LambdaAIChatConfig", + "HyperbolicChatConfig", + "VercelAIGatewayConfig", + "OVHCloudChatConfig", + "OVHCloudEmbeddingConfig", + "CometAPIEmbeddingConfig", + "LemonadeChatConfig", + "SnowflakeEmbeddingConfig", + "AmazonNovaChatConfig", ) # Types that support lazy loading via _lazy_import_types @@ -273,6 +413,146 @@ _LLM_CONFIGS_IMPORT_MAP = { "VoyageRerankConfig": (".llms.voyage.rerank.transformation", "VoyageRerankConfig"), "ClarifaiConfig": (".llms.clarifai.chat.transformation", "ClarifaiConfig"), "AI21ChatConfig": (".llms.ai21.chat.transformation", "AI21ChatConfig"), + "LlamaAPIConfig": (".llms.meta_llama.chat.transformation", "LlamaAPIConfig"), + "TogetherAITextCompletionConfig": (".llms.together_ai.completion.transformation", "TogetherAITextCompletionConfig"), + "CloudflareChatConfig": (".llms.cloudflare.chat.transformation", "CloudflareChatConfig"), + "NovitaConfig": (".llms.novita.chat.transformation", "NovitaConfig"), + "PetalsConfig": (".llms.petals.completion.transformation", "PetalsConfig"), + "OllamaChatConfig": (".llms.ollama.chat.transformation", "OllamaChatConfig"), + "OllamaConfig": (".llms.ollama.completion.transformation", "OllamaConfig"), + "SagemakerConfig": (".llms.sagemaker.completion.transformation", "SagemakerConfig"), + "SagemakerChatConfig": (".llms.sagemaker.chat.transformation", "SagemakerChatConfig"), + "CohereChatConfig": (".llms.cohere.chat.transformation", "CohereChatConfig"), + "AnthropicMessagesConfig": (".llms.anthropic.experimental_pass_through.messages.transformation", "AnthropicMessagesConfig"), + "AmazonAnthropicClaudeMessagesConfig": (".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation", "AmazonAnthropicClaudeMessagesConfig"), + "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"), + "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"), + "VertexGeminiConfig": (".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexGeminiConfig"), + "GoogleAIStudioGeminiConfig": (".llms.gemini.chat.transformation", "GoogleAIStudioGeminiConfig"), + "VertexAIAnthropicConfig": (".llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation", "VertexAIAnthropicConfig"), + "VertexAILlama3Config": (".llms.vertex_ai.vertex_ai_partner_models.llama3.transformation", "VertexAILlama3Config"), + "VertexAIAi21Config": (".llms.vertex_ai.vertex_ai_partner_models.ai21.transformation", "VertexAIAi21Config"), + "AmazonCohereChatConfig": (".llms.bedrock.chat.invoke_handler", "AmazonCohereChatConfig"), + "AmazonBedrockGlobalConfig": (".llms.bedrock.common_utils", "AmazonBedrockGlobalConfig"), + "AmazonAI21Config": (".llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation", "AmazonAI21Config"), + "AmazonInvokeNovaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_nova_transformation", "AmazonInvokeNovaConfig"), + "AmazonQwen2Config": (".llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation", "AmazonQwen2Config"), + "AmazonQwen3Config": (".llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation", "AmazonQwen3Config"), + # Aliases for backwards compatibility + "VertexAIConfig": (".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini", "VertexGeminiConfig"), # Alias + "GeminiConfig": (".llms.gemini.chat.transformation", "GoogleAIStudioGeminiConfig"), # Alias + "AmazonAnthropicConfig": (".llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation", "AmazonAnthropicConfig"), + "AmazonAnthropicClaudeConfig": (".llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation", "AmazonAnthropicClaudeConfig"), + "AmazonCohereConfig": (".llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation", "AmazonCohereConfig"), + "AmazonLlamaConfig": (".llms.bedrock.chat.invoke_transformations.amazon_llama_transformation", "AmazonLlamaConfig"), + "AmazonDeepSeekR1Config": (".llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation", "AmazonDeepSeekR1Config"), + "AmazonMistralConfig": (".llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation", "AmazonMistralConfig"), + "AmazonTitanConfig": (".llms.bedrock.chat.invoke_transformations.amazon_titan_transformation", "AmazonTitanConfig"), + "AmazonTwelveLabsPegasusConfig": (".llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation", "AmazonTwelveLabsPegasusConfig"), + "AmazonInvokeConfig": (".llms.bedrock.chat.invoke_transformations.base_invoke_transformation", "AmazonInvokeConfig"), + "AmazonBedrockOpenAIConfig": (".llms.bedrock.chat.invoke_transformations.amazon_openai_transformation", "AmazonBedrockOpenAIConfig"), + "AmazonStabilityConfig": (".llms.bedrock.image_generation.amazon_stability1_transformation", "AmazonStabilityConfig"), + "AmazonStability3Config": (".llms.bedrock.image_generation.amazon_stability3_transformation", "AmazonStability3Config"), + "AmazonNovaCanvasConfig": (".llms.bedrock.image_generation.amazon_nova_canvas_transformation", "AmazonNovaCanvasConfig"), + "AmazonTitanG1Config": (".llms.bedrock.embed.amazon_titan_g1_transformation", "AmazonTitanG1Config"), + "AmazonTitanMultimodalEmbeddingG1Config": (".llms.bedrock.embed.amazon_titan_multimodal_transformation", "AmazonTitanMultimodalEmbeddingG1Config"), + "CohereV2ChatConfig": (".llms.cohere.chat.v2_transformation", "CohereV2ChatConfig"), + "BedrockCohereEmbeddingConfig": (".llms.bedrock.embed.cohere_transformation", "BedrockCohereEmbeddingConfig"), + "TwelveLabsMarengoEmbeddingConfig": (".llms.bedrock.embed.twelvelabs_marengo_transformation", "TwelveLabsMarengoEmbeddingConfig"), + "AmazonNovaEmbeddingConfig": (".llms.bedrock.embed.amazon_nova_transformation", "AmazonNovaEmbeddingConfig"), + "OpenAIConfig": (".llms.openai.openai", "OpenAIConfig"), + "MistralEmbeddingConfig": (".llms.openai.openai", "MistralEmbeddingConfig"), + "OpenAIImageVariationConfig": (".llms.openai.image_variations.transformation", "OpenAIImageVariationConfig"), + "DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"), + "DeepgramAudioTranscriptionConfig": (".llms.deepgram.audio_transcription.transformation", "DeepgramAudioTranscriptionConfig"), + "TopazImageVariationConfig": (".llms.topaz.image_variations.transformation", "TopazImageVariationConfig"), + "OpenAITextCompletionConfig": ("litellm.llms.openai.completion.transformation", "OpenAITextCompletionConfig"), + "GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"), + "GenAIHubOrchestrationConfig": (".llms.sap.chat.transformation", "GenAIHubOrchestrationConfig"), + "VoyageEmbeddingConfig": (".llms.voyage.embedding.transformation", "VoyageEmbeddingConfig"), + "VoyageContextualEmbeddingConfig": (".llms.voyage.embedding.transformation_contextual", "VoyageContextualEmbeddingConfig"), + "InfinityEmbeddingConfig": (".llms.infinity.embedding.transformation", "InfinityEmbeddingConfig"), + "AzureAIStudioConfig": (".llms.azure_ai.chat.transformation", "AzureAIStudioConfig"), + "MistralConfig": (".llms.mistral.chat.transformation", "MistralConfig"), + "OpenAIResponsesAPIConfig": (".llms.openai.responses.transformation", "OpenAIResponsesAPIConfig"), + "AzureOpenAIResponsesAPIConfig": (".llms.azure.responses.transformation", "AzureOpenAIResponsesAPIConfig"), + "AzureOpenAIOSeriesResponsesAPIConfig": (".llms.azure.responses.o_series_transformation", "AzureOpenAIOSeriesResponsesAPIConfig"), + "XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"), + "LiteLLMProxyResponsesAPIConfig": (".llms.litellm_proxy.responses.transformation", "LiteLLMProxyResponsesAPIConfig"), + "GoogleAIStudioInteractionsConfig": (".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig"), + "OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"), + "AnthropicSkillsConfig": (".llms.anthropic.skills.transformation", "AnthropicSkillsConfig"), + "BaseSkillsAPIConfig": (".llms.base_llm.skills.transformation", "BaseSkillsAPIConfig"), + "GradientAIConfig": (".llms.gradient_ai.chat.transformation", "GradientAIConfig"), + # Alias for backwards compatibility + "OpenAIO1Config": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"), # Alias + "OpenAIGPTConfig": (".llms.openai.chat.gpt_transformation", "OpenAIGPTConfig"), + "OpenAIGPT5Config": (".llms.openai.chat.gpt_5_transformation", "OpenAIGPT5Config"), + "OpenAIWhisperAudioTranscriptionConfig": (".llms.openai.transcriptions.whisper_transformation", "OpenAIWhisperAudioTranscriptionConfig"), + "OpenAIGPTAudioTranscriptionConfig": (".llms.openai.transcriptions.gpt_transformation", "OpenAIGPTAudioTranscriptionConfig"), + "OpenAIGPTAudioConfig": (".llms.openai.chat.gpt_audio_transformation", "OpenAIGPTAudioConfig"), + "NvidiaNimConfig": (".llms.nvidia_nim.chat.transformation", "NvidiaNimConfig"), + "NvidiaNimEmbeddingConfig": (".llms.nvidia_nim.embed", "NvidiaNimEmbeddingConfig"), + "FeatherlessAIConfig": (".llms.featherless_ai.chat.transformation", "FeatherlessAIConfig"), + "CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"), + "BasetenConfig": (".llms.baseten.chat", "BasetenConfig"), + "SambanovaConfig": (".llms.sambanova.chat", "SambanovaConfig"), + "SambaNovaEmbeddingConfig": (".llms.sambanova.embedding.transformation", "SambaNovaEmbeddingConfig"), + "FireworksAIConfig": (".llms.fireworks_ai.chat.transformation", "FireworksAIConfig"), + "FireworksAITextCompletionConfig": (".llms.fireworks_ai.completion.transformation", "FireworksAITextCompletionConfig"), + "FireworksAIAudioTranscriptionConfig": (".llms.fireworks_ai.audio_transcription.transformation", "FireworksAIAudioTranscriptionConfig"), + "FireworksAIEmbeddingConfig": (".llms.fireworks_ai.embed.fireworks_ai_transformation", "FireworksAIEmbeddingConfig"), + "FriendliaiChatConfig": (".llms.friendliai.chat.transformation", "FriendliaiChatConfig"), + "JinaAIEmbeddingConfig": (".llms.jina_ai.embedding.transformation", "JinaAIEmbeddingConfig"), + "XAIChatConfig": (".llms.xai.chat.transformation", "XAIChatConfig"), + "ZAIChatConfig": (".llms.zai.chat.transformation", "ZAIChatConfig"), + "AIMLChatConfig": (".llms.aiml.chat.transformation", "AIMLChatConfig"), + "VolcEngineChatConfig": (".llms.volcengine.chat.transformation", "VolcEngineChatConfig"), + "CodestralTextCompletionConfig": (".llms.codestral.completion.transformation", "CodestralTextCompletionConfig"), + "AzureOpenAIAssistantsAPIConfig": (".llms.azure.azure", "AzureOpenAIAssistantsAPIConfig"), + "HerokuChatConfig": (".llms.heroku.chat.transformation", "HerokuChatConfig"), + "CometAPIConfig": (".llms.cometapi.chat.transformation", "CometAPIConfig"), + "AzureOpenAIConfig": (".llms.azure.chat.gpt_transformation", "AzureOpenAIConfig"), + "AzureOpenAIGPT5Config": (".llms.azure.chat.gpt_5_transformation", "AzureOpenAIGPT5Config"), + "AzureOpenAITextConfig": (".llms.azure.completion.transformation", "AzureOpenAITextConfig"), + "HostedVLLMChatConfig": (".llms.hosted_vllm.chat.transformation", "HostedVLLMChatConfig"), + # Alias for backwards compatibility + "VolcEngineConfig": (".llms.volcengine.chat.transformation", "VolcEngineChatConfig"), # Alias + "LlamafileChatConfig": (".llms.llamafile.chat.transformation", "LlamafileChatConfig"), + "LiteLLMProxyChatConfig": (".llms.litellm_proxy.chat.transformation", "LiteLLMProxyChatConfig"), + "VLLMConfig": (".llms.vllm.completion.transformation", "VLLMConfig"), + "DeepSeekChatConfig": (".llms.deepseek.chat.transformation", "DeepSeekChatConfig"), + "LMStudioChatConfig": (".llms.lm_studio.chat.transformation", "LMStudioChatConfig"), + "LmStudioEmbeddingConfig": (".llms.lm_studio.embed.transformation", "LmStudioEmbeddingConfig"), + "NscaleConfig": (".llms.nscale.chat.transformation", "NscaleConfig"), + "PerplexityChatConfig": (".llms.perplexity.chat.transformation", "PerplexityChatConfig"), + "AzureOpenAIO1Config": (".llms.azure.chat.o_series_transformation", "AzureOpenAIO1Config"), + "IBMWatsonXAIConfig": (".llms.watsonx.completion.transformation", "IBMWatsonXAIConfig"), + "IBMWatsonXChatConfig": (".llms.watsonx.chat.transformation", "IBMWatsonXChatConfig"), + "IBMWatsonXEmbeddingConfig": (".llms.watsonx.embed.transformation", "IBMWatsonXEmbeddingConfig"), + "GenAIHubEmbeddingConfig": (".llms.sap.embed.transformation", "GenAIHubEmbeddingConfig"), + "IBMWatsonXAudioTranscriptionConfig": (".llms.watsonx.audio_transcription.transformation", "IBMWatsonXAudioTranscriptionConfig"), + "GithubCopilotConfig": (".llms.github_copilot.chat.transformation", "GithubCopilotConfig"), + "GithubCopilotResponsesAPIConfig": (".llms.github_copilot.responses.transformation", "GithubCopilotResponsesAPIConfig"), + "GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"), + "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"), + "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"), + "DashScopeChatConfig": (".llms.dashscope.chat.transformation", "DashScopeChatConfig"), + "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), + "DockerModelRunnerChatConfig": (".llms.docker_model_runner.chat.transformation", "DockerModelRunnerChatConfig"), + "V0ChatConfig": (".llms.v0.chat.transformation", "V0ChatConfig"), + "OCIChatConfig": (".llms.oci.chat.transformation", "OCIChatConfig"), + "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), + "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), + "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "HyperbolicChatConfig": (".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig"), + "VercelAIGatewayConfig": (".llms.vercel_ai_gateway.chat.transformation", "VercelAIGatewayConfig"), + "OVHCloudChatConfig": (".llms.ovhcloud.chat.transformation", "OVHCloudChatConfig"), + "OVHCloudEmbeddingConfig": (".llms.ovhcloud.embedding.transformation", "OVHCloudEmbeddingConfig"), + "CometAPIEmbeddingConfig": (".llms.cometapi.embed.transformation", "CometAPIEmbeddingConfig"), + "LemonadeChatConfig": (".llms.lemonade.chat.transformation", "LemonadeChatConfig"), + "SnowflakeEmbeddingConfig": (".llms.snowflake.embedding.transformation", "SnowflakeEmbeddingConfig"), + "AmazonNovaChatConfig": (".llms.amazon_nova.chat.transformation", "AmazonNovaChatConfig"), } # Export all name tuples and import maps for use in _lazy_imports.py diff --git a/litellm/llms/bedrock/image_generation/image_handler.py b/litellm/llms/bedrock/image_generation/image_handler.py index 0a4cde90b27..7270b96ab88 100644 --- a/litellm/llms/bedrock/image_generation/image_handler.py +++ b/litellm/llms/bedrock/image_generation/image_handler.py @@ -12,6 +12,9 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( AmazonNovaCanvasConfig, ) +from litellm.llms.bedrock.image_generation.amazon_stability1_transformation import ( + AmazonStabilityConfig, +) from litellm.llms.bedrock.image_generation.amazon_stability3_transformation import ( AmazonStability3Config, ) @@ -50,7 +53,7 @@ BedrockImageConfigClass = Union[ type[AmazonTitanImageGenerationConfig], type[AmazonNovaCanvasConfig], type[AmazonStability3Config], - type[litellm.AmazonStabilityConfig], + type[AmazonStabilityConfig], ] From 87337abfea37ccbddfd2e4188d0ecfe188318cc7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 13:52:05 -0800 Subject: [PATCH 046/388] Support Disable Admin UI --- .../ui_discovery_endpoints.py | 2 + .../ui_discovery_endpoints.py | 1 + .../test_ui_discovery_endpoints.py | 41 ++++++++ .../(dashboard)/hooks/useAuthorized.test.ts | 96 ++++++++++++++++--- .../app/(dashboard)/hooks/useAuthorized.ts | 17 ++-- .../src/app/login/LoginPage.test.tsx | 23 +++++ .../src/app/login/LoginPage.tsx | 38 ++++++++ .../src/components/networking.tsx | 1 + 8 files changed, 202 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py index 9aaa2fb8381..cbe28849b1e 100644 --- a/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -18,9 +18,11 @@ async def get_ui_config(): from litellm.proxy.auth.auth_utils import _has_user_setup_sso auto_redirect_ui_login_to_sso = os.getenv("AUTO_REDIRECT_UI_LOGIN_TO_SSO", "true").lower() == "true" + admin_ui_disabled = os.getenv("DISABLE_ADMIN_UI", "false").lower() == "true" return UiDiscoveryEndpoints( server_root_path=get_server_root_path(), proxy_base_url=get_proxy_base_url(), auto_redirect_to_sso=_has_user_setup_sso() and auto_redirect_ui_login_to_sso, + admin_ui_disabled=admin_ui_disabled, ) diff --git a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py index f100dd35fa6..dc167667bc0 100644 --- a/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py +++ b/litellm/types/proxy/discovery_endpoints/ui_discovery_endpoints.py @@ -7,3 +7,4 @@ class UiDiscoveryEndpoints(BaseModel): server_root_path: str proxy_base_url: Optional[str] auto_redirect_to_sso: bool + admin_ui_disabled: bool diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index 599d5437589..1d31d4f7bf1 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -30,6 +30,7 @@ def test_ui_discovery_endpoints_with_defaults(): assert data["server_root_path"] == "/" assert data["proxy_base_url"] is None assert data["auto_redirect_to_sso"] is False + assert data["admin_ui_disabled"] is False def test_ui_discovery_endpoints_with_custom_server_root_path(): @@ -144,3 +145,43 @@ def test_ui_discovery_endpoints_both_routes_return_same_data(): assert response2.status_code == 200 assert response1.json() == response2.json() + +def test_ui_discovery_endpoints_with_admin_ui_disabled(): + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ + patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ + patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "true"}, clear=False): + + response = client.get("/.well-known/litellm-ui-config") + + assert response.status_code == 200 + data = response.json() + assert data["server_root_path"] == "/" + assert data["proxy_base_url"] is None + assert data["auto_redirect_to_sso"] is False + assert data["admin_ui_disabled"] is True + + +def test_ui_discovery_endpoints_with_admin_ui_enabled(): + app = FastAPI() + app.include_router(router) + client = TestClient(app) + + with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ + patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ + patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): + + response = client.get("/.well-known/litellm-ui-config") + + assert response.status_code == 200 + data = response.json() + assert data["server_root_path"] == "/" + assert data["proxy_base_url"] is None + assert data["auto_redirect_to_sso"] is False + assert data["admin_ui_disabled"] is False + diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts index 9198450a63d..26684619378 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts @@ -1,12 +1,15 @@ /* @vitest-environment jsdom */ -import { renderHook } from "@testing-library/react"; +import React from "react"; +import { renderHook, waitFor } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import useAuthorized from "./useAuthorized"; -const { replaceMock, clearTokenCookiesMock, getProxyBaseUrlMock } = vi.hoisted(() => ({ +const { replaceMock, clearTokenCookiesMock, getProxyBaseUrlMock, getUiConfigMock } = vi.hoisted(() => ({ replaceMock: vi.fn(), clearTokenCookiesMock: vi.fn(), getProxyBaseUrlMock: vi.fn(() => "http://proxy.example"), + getUiConfigMock: vi.fn(), })); vi.mock("next/navigation", () => ({ @@ -15,9 +18,14 @@ vi.mock("next/navigation", () => ({ }), })); -vi.mock("@/components/networking", () => ({ - getProxyBaseUrl: getProxyBaseUrlMock, -})); +vi.mock("@/components/networking", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + getProxyBaseUrl: getProxyBaseUrlMock, + getUiConfig: getUiConfigMock, + }; +}); vi.mock("@/utils/cookieUtils", async (importOriginal) => { const actual = await importOriginal(); @@ -27,6 +35,21 @@ vi.mock("@/utils/cookieUtils", async (importOriginal) => { }; }); +const createQueryClient = () => + new QueryClient({ + defaultOptions: { + queries: { + retry: false, + gcTime: 0, + }, + }, + }); + +const wrapper = ({ children }: { children: React.ReactNode }) => { + const queryClient = createQueryClient(); + return React.createElement(QueryClientProvider, { client: queryClient }, children); +}; + const createJwt = (payload: Record) => { const base64Url = btoa(JSON.stringify(payload)).replace(/=+$/, "").replace(/\+/g, "-").replace(/\//g, "_"); return `eyJhbGciOiJub25lIn0.${base64Url}.signature`; @@ -41,10 +64,18 @@ describe("useAuthorized", () => { replaceMock.mockReset(); clearTokenCookiesMock.mockReset(); getProxyBaseUrlMock.mockClear(); + getUiConfigMock.mockReset(); clearCookie(); }); - it("should decode the token and expose user details", () => { + it("should decode the token and expose user details", async () => { + getUiConfigMock.mockResolvedValue({ + server_root_path: "/", + proxy_base_url: null, + auto_redirect_to_sso: false, + admin_ui_disabled: false, + }); + const token = createJwt({ key: "api-key-123", user_id: "user-1", @@ -56,9 +87,12 @@ describe("useAuthorized", () => { }); document.cookie = `token=${token}; path=/;`; - const { result } = renderHook(() => useAuthorized()); + const { result } = renderHook(() => useAuthorized(), { wrapper }); + + await waitFor(() => { + expect(result.current.token).toBe(token); + }); - expect(result.current.token).toBe(token); expect(result.current.accessToken).toBe("api-key-123"); expect(result.current.userId).toBe("user-1"); expect(result.current.userEmail).toBe("user@example.com"); @@ -69,14 +103,54 @@ describe("useAuthorized", () => { expect(replaceMock).not.toHaveBeenCalled(); }); - it("should clear cookies and redirect on an invalid token", () => { + it("should clear cookies and redirect on an invalid token", async () => { + getUiConfigMock.mockResolvedValue({ + server_root_path: "/", + proxy_base_url: null, + auto_redirect_to_sso: false, + admin_ui_disabled: false, + }); + document.cookie = "token=invalid-token; path=/;"; - const { result } = renderHook(() => useAuthorized()); + const { result } = renderHook(() => useAuthorized(), { wrapper }); + + await waitFor(() => { + expect(clearTokenCookiesMock).toHaveBeenCalled(); + }); - expect(clearTokenCookiesMock).toHaveBeenCalled(); expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); expect(result.current.accessToken).toBeNull(); expect(result.current.userRole).toBe("Undefined Role"); }); + + it("should redirect even with valid token if admin_ui_disabled is true", async () => { + getUiConfigMock.mockResolvedValue({ + server_root_path: "/", + proxy_base_url: null, + auto_redirect_to_sso: false, + admin_ui_disabled: true, + }); + + const token = createJwt({ + key: "api-key-123", + user_id: "user-1", + user_email: "user@example.com", + user_role: "app_admin", + premium_user: true, + disabled_non_admin_personal_key_creation: false, + login_method: "username_password", + }); + document.cookie = `token=${token}; path=/;`; + + const { result } = renderHook(() => useAuthorized(), { wrapper }); + + await waitFor(() => { + expect(replaceMock).toHaveBeenCalledWith("http://proxy.example/ui/login"); + }); + + expect(result.current.accessToken).toBe("api-key-123"); + expect(result.current.userId).toBe("user-1"); + expect(result.current.userEmail).toBe("user@example.com"); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts index 7610c6346be..62d514f0668 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.ts @@ -1,10 +1,11 @@ "use client"; -import { useEffect, useMemo } from "react"; -import { useRouter } from "next/navigation"; -import { jwtDecode } from "jwt-decode"; -import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; import { getProxyBaseUrl } from "@/components/networking"; +import { clearTokenCookies, getCookie } from "@/utils/cookieUtils"; +import { jwtDecode } from "jwt-decode"; +import { useRouter } from "next/navigation"; +import { useEffect, useMemo } from "react"; +import { useUIConfig } from "./uiConfig/useUIConfig"; function formatUserRole(userRole: string) { if (!userRole) { @@ -37,15 +38,19 @@ function formatUserRole(userRole: string) { const useAuthorized = () => { const router = useRouter(); + const { data: uiConfig, isLoading: isUIConfigLoading } = useUIConfig(); const token = typeof document !== "undefined" ? getCookie("token") : null; // Redirect after mount if missing/invalid token useEffect(() => { - if (!token) { + if (isUIConfigLoading) { + return; + } + if (!token || uiConfig?.admin_ui_disabled) { router.replace(`${getProxyBaseUrl()}/ui/login`); } - }, [token, router]); + }, [token, router, isUIConfigLoading, uiConfig]); // Decode safely const decoded = useMemo(() => { diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx index cce063eceb7..79834512605 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.test.tsx @@ -169,4 +169,27 @@ describe("LoginPage", () => { expect(mockPush).not.toHaveBeenCalled(); }); + + it("should show alert when admin_ui_disabled is true", async () => { + (useUIConfig as ReturnType).mockReturnValue({ + data: { admin_ui_disabled: true, server_root_path: "/", proxy_base_url: null }, + isLoading: false, + }); + (getCookie as ReturnType).mockReturnValue(null); + + const queryClient = createQueryClient(); + render( + + + , + ); + + await waitFor(() => { + expect(screen.getByRole("alert")).toBeInTheDocument(); + expect(screen.getByText("Admin UI Disabled")).toBeInTheDocument(); + }); + + expect(mockPush).not.toHaveBeenCalled(); + expect(mockReplace).not.toHaveBeenCalled(); + }); }); diff --git a/ui/litellm-dashboard/src/app/login/LoginPage.tsx b/ui/litellm-dashboard/src/app/login/LoginPage.tsx index 85f2c6dd870..620cb41dfee 100644 --- a/ui/litellm-dashboard/src/app/login/LoginPage.tsx +++ b/ui/litellm-dashboard/src/app/login/LoginPage.tsx @@ -25,6 +25,12 @@ function LoginPageContent() { return; } + // Check if admin UI is disabled + if (uiConfig && uiConfig.admin_ui_disabled) { + setIsLoading(false); + return; + } + const rawToken = getCookie("token"); if (rawToken && !isJwtExpired(rawToken)) { router.replace(`${getProxyBaseUrl()}/ui`); @@ -59,6 +65,38 @@ function LoginPageContent() { return ; } + // Show disabled message if admin UI is disabled + if (uiConfig && uiConfig.admin_ui_disabled) { + return ( +

+ ); + } + return (
diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 21f18168144..aeba207db26 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -230,6 +230,7 @@ export interface LiteLLMWellKnownUiConfig { server_root_path: string; proxy_base_url: string | null; auto_redirect_to_sso: boolean; + admin_ui_disabled: boolean; } export interface CredentialsResponse { From 4d005fe1c48f525ca9a25c412805b2e6a371afae Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Tue, 23 Dec 2025 13:55:24 -0800 Subject: [PATCH 047/388] [Refactor]: Lazy load additional components (types, callbacks, utilities) (#18396) --- litellm/__init__.py | 152 ++++++++++++++---- litellm/_lazy_imports_registry.py | 19 ++- .../litellm_core_utils/default_encoding.py | 19 ++- 3 files changed, 153 insertions(+), 37 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index ecb101e2de5..02d7fef749b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -74,39 +74,24 @@ from litellm.constants import ( DEFAULT_SOFT_BUDGET, DEFAULT_ALLOWED_FAILS, ) -from litellm.types.secret_managers.main import ( - KeyManagementSystem, - KeyManagementSettings, -) -from litellm.types.proxy.management_endpoints.ui_sso import ( - DefaultTeamSSOParams, - LiteLLM_UpperboundKeyGenerateParams, -) -from litellm.types.utils import LlmProviders -from litellm.types.utils import PriorityReservationSettings -from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager import httpx import dotenv -from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup +# register_async_client_cleanup is lazy-loaded and called on first access litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV" if litellm_mode == "DEV": dotenv.load_dotenv() - -# Register async client cleanup to prevent resource leaks -register_async_client_cleanup() #################################################### if set_verbose: _turn_on_debug() #################################################### ### Callbacks /Logging / Success / Failure Handlers ##### -CALLBACK_TYPES = Union[str, Callable, CustomLogger] +CALLBACK_TYPES = Union[str, Callable, "CustomLogger"] # CustomLogger is lazy-loaded input_callback: List[CALLBACK_TYPES] = [] success_callback: List[CALLBACK_TYPES] = [] failure_callback: List[CALLBACK_TYPES] = [] service_callback: List[CALLBACK_TYPES] = [] -logging_callback_manager = LoggingCallbackManager() +# logging_callback_manager is lazy-loaded via __getattr__ _custom_logger_compatible_callbacks_literal = Literal[ "lago", "openmeter", @@ -158,7 +143,7 @@ _known_custom_logger_compatible_callbacks: List = list( get_args(_custom_logger_compatible_callbacks_literal) ) callbacks: List[ - Union[Callable, _custom_logger_compatible_callbacks_literal, CustomLogger] + Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded ] = [] callback_settings: Dict[str, Dict[str, Any]] = {} initialized_langfuse_clients: int = 0 @@ -175,13 +160,13 @@ generic_api_use_v1: Optional[bool] = ( False # if you want to use v1 generic api logged payload ) argilla_transformation_object: Optional[Dict[str, Any]] = None -_async_input_callback: List[Union[str, Callable, CustomLogger]] = ( +_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_success_callback: List[Union[str, Callable, CustomLogger]] = ( +_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. -_async_failure_callback: List[Union[str, Callable, CustomLogger]] = ( +_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded [] ) # internal variable - async custom callbacks are routed here. pre_call_rules: List[Callable] = [] @@ -388,9 +373,7 @@ public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {} priority_reservation: Optional[ Dict[str, Union[float, "PriorityReservationDict"]] ] = None -priority_reservation_settings: "PriorityReservationSettings" = ( - PriorityReservationSettings() -) +# priority_reservation_settings is lazy-loaded via __getattr__ ######## Networking Settings ######## @@ -423,8 +406,11 @@ secret_manager_client: Optional[Any] = ( None # list of instantiated key management clients - e.g. azure kv, infisical, etc. ) _google_kms_resource_name: Optional[str] = None -_key_management_system: Optional[KeyManagementSystem] = None -_key_management_settings: KeyManagementSettings = KeyManagementSettings() +_key_management_system: Optional["KeyManagementSystem"] = None +# Note: KeyManagementSettings must be eagerly imported because _key_management_settings +# is accessed during import time in secret_managers/main.py +# We'll import it after the lazy import system is set up +# We can't define it here because KeyManagementSettings is lazy-loaded #### PII MASKING #### output_parse_pii: bool = False ############################################# @@ -920,7 +906,7 @@ model_list = list( model_list_set = set(model_list) -provider_list: List[Union[LlmProviders, str]] = list(LlmProviders) +# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time models_by_provider: dict = { @@ -1055,9 +1041,15 @@ openai_image_generation_models = ["dall-e-2", "dall-e-3"] ####### VIDEO GENERATION MODELS ################### openai_video_generation_models = ["sora-2"] -from .timeout import timeout +# timeout is lazy-loaded via __getattr__ from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls + +# Import KeyManagementSettings here (before utils import) because _key_management_settings +# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils) +from litellm.types.secret_managers.main import KeyManagementSettings +_key_management_settings: KeyManagementSettings = KeyManagementSettings() + # client must be imported immediately as it's used as a decorator at function definition time from .utils import client # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py @@ -1079,9 +1071,6 @@ from .llms.vertex_ai.vertex_embeddings.transformation import ( vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig() -from .llms.bedrock.chat.invoke_handler import ( - bedrock_tool_name_mappings, -) from .llms.bedrock.embed.amazon_titan_v2_transformation import ( AmazonTitanV2Config, @@ -1091,12 +1080,13 @@ from .llms.topaz.common_utils import TopazModelInfo # OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo -from .llms.azure.azure import ( - AzureOpenAIError, -) # PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) # All remaining configs are now lazy loaded - see _lazy_imports_registry.py +# Import LlmProviders here (before main import) because it's imported during import time +# in multiple places including openai.py (via main import) +from litellm.types.utils import LlmProviders + ## Lazy loading this is not straightforward, will leave it here for now. from .main import * # type: ignore @@ -1462,6 +1452,10 @@ if TYPE_CHECKING: StandardKeyGenerationConfig, ) from litellm.types.guardrails import GuardrailItem + from litellm.types.proxy.management_endpoints.ui_sso import ( + DefaultTeamSSOParams, + LiteLLM_UpperboundKeyGenerateParams, + ) # Cost calculator functions cost_per_token: Callable[..., Tuple[float, float]] @@ -1506,11 +1500,46 @@ if TYPE_CHECKING: module_level_aclient: AsyncHTTPHandler module_level_client: HTTPHandler + # Bedrock tool name mappings instance (lazy-loaded) + from litellm.caching.caching import InMemoryCache + bedrock_tool_name_mappings: InMemoryCache + + # Azure exception class (lazy-loaded) + from litellm.llms.azure.common_utils import AzureOpenAIError + + # Secret manager types (lazy-loaded) + from litellm.types.secret_managers.main import ( + KeyManagementSystem, + KeyManagementSettings, # Not lazy-loaded - needed for _key_management_settings initialization + ) + + # Custom logger class (lazy-loaded) + from litellm.integrations.custom_logger import CustomLogger + + # Logging callback manager class and instance (lazy-loaded) + from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + logging_callback_manager: LoggingCallbackManager + + # provider_list is lazy-loaded + from litellm.types.utils import LlmProviders + provider_list: List[Union[LlmProviders, str]] + # Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block +# Track if async client cleanup has been registered (for lazy loading) +_async_client_cleanup_registered = False + + def __getattr__(name: str) -> Any: """Lazy import handler with cached registry for improved performance.""" + global _async_client_cleanup_registered + # Register async client cleanup on first access (only once) + if not _async_client_cleanup_registered: + from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup + register_async_client_cleanup() + _async_client_cleanup_registered = True + # Use cached registry from _lazy_imports instead of importing tuples every time from ._lazy_imports import _get_lazy_import_registry @@ -1531,6 +1560,26 @@ def __getattr__(name: str) -> Any: _globals["encoding"] = _encoding return _globals["encoding"] + # Lazy load bedrock_tool_name_mappings instance + if name == "bedrock_tool_name_mappings": + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + # Check if already cached + if "bedrock_tool_name_mappings" not in _globals: + from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings + _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings + return _globals["bedrock_tool_name_mappings"] + + # Lazy load AzureOpenAIError exception class + if name == "AzureOpenAIError": + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + # Check if already cached + if "AzureOpenAIError" not in _globals: + from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError + _globals["AzureOpenAIError"] = _AzureOpenAIError + return _globals["AzureOpenAIError"] + # Lazy load openaiOSeriesConfig instance if name == "openaiOSeriesConfig": from ._lazy_imports import _get_litellm_globals @@ -1561,6 +1610,39 @@ def __getattr__(name: str) -> Any: # Handle OpenAIO1Config alias if name == "OpenAIO1Config": return __getattr__("OpenAIOSeriesConfig") + + # Lazy load provider_list + if name == "provider_list": + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + # Check if already cached + if "provider_list" not in _globals: + # LlmProviders is eagerly imported above, so we can import it directly + from litellm.types.utils import LlmProviders + _globals["provider_list"] = list(LlmProviders) + return _globals["provider_list"] + + # Lazy load priority_reservation_settings instance + if name == "priority_reservation_settings": + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + # Check if already cached + if "priority_reservation_settings" not in _globals: + # Import the class and instantiate it + PriorityReservationSettings = __getattr__("PriorityReservationSettings") + _globals["priority_reservation_settings"] = PriorityReservationSettings() + return _globals["priority_reservation_settings"] + + # Lazy load logging_callback_manager instance + if name == "logging_callback_manager": + from ._lazy_imports import _get_litellm_globals + _globals = _get_litellm_globals() + # Check if already cached + if "logging_callback_manager" not in _globals: + # Import the class and instantiate it + LoggingCallbackManager = __getattr__("LoggingCallbackManager") + _globals["logging_callback_manager"] = LoggingCallbackManager() + return _globals["logging_callback_manager"] raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 31ef0cf69ff..e2f80a14391 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -31,7 +31,7 @@ UTILS_NAMES = ( "get_supported_openai_params", "get_api_base", "get_first_chars_messages", "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", - "ModelResponseListIterator", "get_valid_models", + "ModelResponseListIterator", "get_valid_models", "timeout", ) # Token counter names that support lazy loading via _lazy_import_token_counter @@ -275,6 +275,16 @@ LLM_CONFIG_NAMES = ( # Types that support lazy loading via _lazy_import_types TYPES_NAMES = ( "GuardrailItem", + "DefaultTeamSSOParams", + "LiteLLM_UpperboundKeyGenerateParams", + "KeyManagementSystem", + "PriorityReservationSettings", + "CustomLogger", + "LoggingCallbackManager", + # Note: LlmProviders is NOT lazy-loaded because it's imported during import time + # in multiple places including openai.py (via main import) + # Note: KeyManagementSettings is NOT lazy-loaded because _key_management_settings + # is accessed during import time in secret_managers/main.py ) # Import maps for registry pattern - reduces repetition @@ -319,6 +329,7 @@ _UTILS_IMPORT_MAP = { "get_provider_fields": (".utils", "get_provider_fields"), "ModelResponseListIterator": (".utils", "ModelResponseListIterator"), "get_valid_models": (".utils", "get_valid_models"), + "timeout": (".timeout", "timeout"), } _COST_CALCULATOR_IMPORT_MAP = { @@ -367,6 +378,12 @@ _DOTPROMPT_IMPORT_MAP = { _TYPES_IMPORT_MAP = { "GuardrailItem": ("litellm.types.guardrails", "GuardrailItem"), + "DefaultTeamSSOParams": ("litellm.types.proxy.management_endpoints.ui_sso", "DefaultTeamSSOParams"), + "LiteLLM_UpperboundKeyGenerateParams": ("litellm.types.proxy.management_endpoints.ui_sso", "LiteLLM_UpperboundKeyGenerateParams"), + "KeyManagementSystem": ("litellm.types.secret_managers.main", "KeyManagementSystem"), + "PriorityReservationSettings": ("litellm.types.utils", "PriorityReservationSettings"), + "CustomLogger": ("litellm.integrations.custom_logger", "CustomLogger"), + "LoggingCallbackManager": ("litellm.litellm_core_utils.logging_callback_manager", "LoggingCallbackManager"), } _LLM_CONFIGS_IMPORT_MAP = { diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index 93b3132912c..d1f51e50720 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -19,5 +19,22 @@ os.environ["TIKTOKEN_CACHE_DIR"] = os.getenv( "CUSTOM_TIKTOKEN_CACHE_DIR", filename ) # use local copy of tiktoken b/c of - https://github.com/BerriAI/litellm/issues/1071 import tiktoken +import time +import random -encoding = tiktoken.get_encoding("cl100k_base") +# Retry logic to handle race conditions when multiple processes try to create +# the tiktoken cache file simultaneously (common in parallel test execution on Windows) +_max_retries = 5 +_retry_delay = 0.1 # Start with 100ms + +for attempt in range(_max_retries): + try: + encoding = tiktoken.get_encoding("cl100k_base") + break + except (FileExistsError, OSError) as e: + if attempt == _max_retries - 1: + # Last attempt, re-raise the exception + raise + # Exponential backoff with jitter to reduce collision probability + delay = _retry_delay * (2 ** attempt) + random.uniform(0, 0.1) + time.sleep(delay) From d9bcc33ba70b8ad1d0ba2ebfc5bfdb1fa983dc0d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 14:10:41 -0800 Subject: [PATCH 048/388] Fixing tests --- .../test_ui_discovery_endpoints.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py index 1d31d4f7bf1..88d31e993dd 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_ui_discovery_endpoints.py @@ -21,7 +21,7 @@ def test_ui_discovery_endpoints_with_defaults(): with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ - patch.dict(os.environ, {}, clear=False): + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): response = client.get("/.well-known/litellm-ui-config") @@ -41,7 +41,7 @@ def test_ui_discovery_endpoints_with_custom_server_root_path(): with patch("litellm.proxy.utils.get_server_root_path", return_value="/litellm"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ - patch.dict(os.environ, {}, clear=False): + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): response = client.get("/.well-known/litellm-ui-config") @@ -60,7 +60,7 @@ def test_ui_discovery_endpoints_with_proxy_base_url_when_set(): with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com"), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ - patch.dict(os.environ, {}, clear=False): + patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}, clear=False): response = client.get("/litellm/.well-known/litellm-ui-config") @@ -79,7 +79,7 @@ def test_ui_discovery_endpoints_with_sso_configured_and_auto_redirect_enabled(): with patch("litellm.proxy.utils.get_server_root_path", return_value="/litellm"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com"), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), \ - patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true"}, clear=False): + patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, clear=False): response = client.get("/.well-known/litellm-ui-config") @@ -98,7 +98,7 @@ def test_ui_discovery_endpoints_with_sso_configured_but_auto_redirect_disabled() with patch("litellm.proxy.utils.get_server_root_path", return_value="/litellm"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com"), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), \ - patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "false"}, clear=False): + patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "false", "DISABLE_ADMIN_UI": "false"}, clear=False): response = client.get("/.well-known/litellm-ui-config") @@ -117,7 +117,7 @@ def test_ui_discovery_endpoints_with_sso_not_configured_but_auto_redirect_enable with patch("litellm.proxy.utils.get_server_root_path", return_value="/"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value=None), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=False), \ - patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true"}, clear=False): + patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, clear=False): response = client.get("/.well-known/litellm-ui-config") @@ -136,7 +136,7 @@ def test_ui_discovery_endpoints_both_routes_return_same_data(): with patch("litellm.proxy.utils.get_server_root_path", return_value="/litellm"), \ patch("litellm.proxy.utils.get_proxy_base_url", return_value="https://proxy.example.com"), \ patch("litellm.proxy.auth.auth_utils._has_user_setup_sso", return_value=True), \ - patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true"}, clear=False): + patch.dict(os.environ, {"AUTO_REDIRECT_UI_LOGIN_TO_SSO": "true", "DISABLE_ADMIN_UI": "false"}, clear=False): response1 = client.get("/.well-known/litellm-ui-config") response2 = client.get("/litellm/.well-known/litellm-ui-config") From 6bb5254c9b92d3a9e9cf9826bc0e9f101a82435b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 15:08:07 -0800 Subject: [PATCH 049/388] Revert "[Fix] UI - Disappears in Development Environments" --- docker/Dockerfile.non_root | 24 ++--- litellm/proxy/proxy_server.py | 59 ++++++------ tests/test_litellm/proxy/test_proxy_server.py | 90 +++++-------------- 3 files changed, 62 insertions(+), 111 deletions(-) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index af1bb5b2022..7e9147a124e 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -40,7 +40,7 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI using the upstream command order while keeping a single RUN layer -RUN mkdir -p /var/lib/litellm/ui && \ +RUN mkdir -p /tmp/litellm_ui && \ npm install -g npm@latest && npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ @@ -49,10 +49,10 @@ RUN mkdir -p /var/lib/litellm/ui && \ rm -f package-lock.json && \ npm install --legacy-peer-deps && \ npm run build && \ - cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ - mkdir -p /var/lib/litellm/assets && \ - cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ - ( cd /var/lib/litellm/ui && \ + cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \ + mkdir -p /tmp/litellm_assets && \ + cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \ + ( cd /tmp/litellm_ui && \ for html_file in *.html; do \ if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ folder_name="${html_file%.html}" && \ @@ -111,8 +111,8 @@ COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /ap COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf COPY --from=builder /app/schema.prisma /app/ COPY --from=builder /wheels/ /wheels/ -COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui -COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets +COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui +COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets COPY --from=builder /app/.cache /app/.cache COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras COPY --from=builder \ @@ -145,8 +145,8 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ # Permissions, cleanup, and Prisma prep RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ - mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \ - chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \ + mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \ + chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \ pip uninstall jwt -y || true && \ pip uninstall PyJWT -y || true && \ pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \ @@ -156,11 +156,11 @@ RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \ LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ + chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ + chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ + chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \ chmod -R g+rX $PRISMA_PATH && \ chmod -R g+rX /app/.cache && \ diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a1be8153b29..f754e52796f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -946,19 +946,20 @@ try: # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) # and ensures extensionless routes like /ui/login work via /index.html. is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + runtime_ui_path = "/tmp/litellm_ui" - # Only use runtime UI path in Docker/non-root environments - # In local development, use the packaged UI directly - if is_non_root: - # Use /var/lib/litellm/ui for Docker (more secure than /tmp) - runtime_ui_path = "/var/lib/litellm/ui" - - if _dir_has_content(runtime_ui_path): + if _dir_has_content(runtime_ui_path): + if is_non_root: verbose_proxy_logger.info( f"Using pre-built UI for non-root Docker: {runtime_ui_path}" ) - ui_path = runtime_ui_path else: + verbose_proxy_logger.info( + f"Using cached runtime UI directory: {runtime_ui_path}" + ) + ui_path = runtime_ui_path + else: + if is_non_root: verbose_proxy_logger.error( f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." ) @@ -966,32 +967,32 @@ try: f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" ) - try: - os.makedirs(runtime_ui_path, exist_ok=True) - if not _dir_has_content(runtime_ui_path) and _dir_has_content( - packaged_ui_path - ): - shutil.copytree( - packaged_ui_path, - runtime_ui_path, - dirs_exist_ok=True, - ) - except Exception as e: + try: + os.makedirs(runtime_ui_path, exist_ok=True) + if not _dir_has_content(runtime_ui_path) and _dir_has_content( + packaged_ui_path + ): + shutil.copytree( + packaged_ui_path, + runtime_ui_path, + dirs_exist_ok=True, + ) + except Exception as e: + if is_non_root: verbose_proxy_logger.exception( f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" ) - else: - if _dir_has_content(runtime_ui_path): + else: + if _dir_has_content(runtime_ui_path): + if is_non_root: verbose_proxy_logger.info( f"Using populated UI for non-root Docker: {runtime_ui_path}" ) - ui_path = runtime_ui_path - else: - # Local development: use packaged UI directly, no runtime copy needed - verbose_proxy_logger.info( - f"Using packaged UI directory for local development: {packaged_ui_path}" - ) - ui_path = packaged_ui_path + else: + verbose_proxy_logger.info( + f"Using populated runtime UI directory: {runtime_ui_path}" + ) + ui_path = runtime_ui_path # Only modify files if a custom server root path is set if server_root_path and server_root_path != "/": @@ -8884,7 +8885,7 @@ def get_image(): default_site_logo = os.path.join(current_dir, "logo.jpg") is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir + assets_dir = "/tmp/litellm_assets" if is_non_root else current_dir if is_non_root: os.makedirs(assets_dir, exist_ok=True) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 737877b0ed2..fd7036b9940 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -273,10 +273,6 @@ def test_sso_key_generate_shows_deprecation_banner(client_no_auth, monkeypatch): def test_restructure_ui_html_files_handles_nested_routes(tmp_path): - """ - Test that _restructure_ui_html_files correctly restructures HTML files. - Note: This function is only called when is_non_root is True (ui_path != packaged_ui_path). - """ from litellm.proxy import proxy_server ui_root = tmp_path / "ui" @@ -310,10 +306,7 @@ def test_restructure_ui_html_files_handles_nested_routes(tmp_path): def test_ui_extensionless_route_requires_restructure(tmp_path): - """ - Regression for non-root fallback: /ui/login expects login/index.html. - Note: Restructuring only happens when is_non_root is True (ui_path != packaged_ui_path). - """ + """Regression for non-root fallback: /ui/login expects login/index.html.""" from litellm.proxy import proxy_server @@ -338,49 +331,6 @@ def test_ui_extensionless_route_requires_restructure(tmp_path): assert "login" in response.text -def test_restructure_only_happens_when_non_root(monkeypatch): - """ - Test that restructuring logic only executes when LITELLM_NON_ROOT is true. - When is_non_root is False, ui_path == packaged_ui_path, so restructuring is skipped. - """ - # Test Case 1: is_non_root is True - ui_path != packaged_ui_path, so restructuring should happen - monkeypatch.setenv("LITELLM_NON_ROOT", "true") - - runtime_ui_path = "/var/lib/litellm/ui" - packaged_ui_path = "/some/packaged/ui/path" - - # Simulate the logic from proxy_server.py - is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - if is_non_root: - ui_path = runtime_ui_path - else: - ui_path = packaged_ui_path - - # This is the condition that determines if restructuring happens - should_restructure = ui_path != packaged_ui_path - - assert is_non_root is True - assert should_restructure is True - assert ui_path == runtime_ui_path - - # Test Case 2: is_non_root is False - ui_path == packaged_ui_path, so restructuring should NOT happen - monkeypatch.delenv("LITELLM_NON_ROOT", raising=False) - - # Simulate the logic from proxy_server.py - is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - if is_non_root: - ui_path = runtime_ui_path - else: - ui_path = packaged_ui_path - - # This is the condition that determines if restructuring happens - should_restructure = ui_path != packaged_ui_path - - assert is_non_root is False - assert should_restructure is False - assert ui_path == packaged_ui_path - - @pytest.mark.asyncio async def test_initialize_scheduled_jobs_credentials(monkeypatch): """ @@ -575,7 +525,7 @@ mock_prisma = MockPrisma() @pytest.mark.asyncio async def test_aaaproxy_startup_master_key(mock_prisma, monkeypatch, tmp_path): """ - Test that master_key is correctly loaded from either config.yaml or environment variables. + Test that master_key is correctly loaded from either config.yaml or environment variables """ import yaml from fastapi import FastAPI @@ -2906,9 +2856,9 @@ def test_root_redirect_when_docs_url_not_root_and_redirect_url_set(monkeypatch): assert response.headers["location"] == test_redirect_url -def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): +def test_get_image_non_root_uses_tmp_assets_dir(monkeypatch): """ - Test that get_image uses /var/lib/litellm/assets when LITELLM_NON_ROOT is true. + Test that get_image uses /tmp/litellm_assets when LITELLM_NON_ROOT is true. """ from unittest.mock import patch @@ -2937,14 +2887,14 @@ def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): # Call the function get_image() - # Verify makedirs was called with /var/lib/litellm/assets - mock_makedirs.assert_called_once_with("/var/lib/litellm/assets", exist_ok=True) + # Verify makedirs was called with /tmp/litellm_assets + mock_makedirs.assert_called_once_with("/tmp/litellm_assets", exist_ok=True) def test_get_image_non_root_fallback_to_default_logo(monkeypatch): """ Test that get_image falls back to default_site_logo when logo doesn't exist - in /var/lib/litellm/assets for non-root case. + in /tmp/litellm_assets for non-root case. """ from unittest.mock import patch @@ -2954,13 +2904,13 @@ def test_get_image_non_root_fallback_to_default_logo(monkeypatch): monkeypatch.setenv("LITELLM_NON_ROOT", "true") monkeypatch.delenv("UI_LOGO_PATH", raising=False) - # Track path.exists calls to verify it checks /var/lib/litellm/assets/logo.jpg + # Track path.exists calls to verify it checks /tmp/litellm_assets/logo.jpg exists_calls = [] def exists_side_effect(path): exists_calls.append(path) - # Return False for /var/lib/litellm/assets/logo.jpg to trigger fallback - if "/var/lib/litellm/assets/logo.jpg" in path: + # Return False for /tmp/litellm_assets/logo.jpg to trigger fallback + if "/tmp/litellm_assets/logo.jpg" in path: return False return True @@ -2983,13 +2933,13 @@ def test_get_image_non_root_fallback_to_default_logo(monkeypatch): # Call the function get_image() - # Verify makedirs was called with /var/lib/litellm/assets - mock_makedirs.assert_called_once_with("/var/lib/litellm/assets", exist_ok=True) + # Verify makedirs was called with /tmp/litellm_assets + mock_makedirs.assert_called_once_with("/tmp/litellm_assets", exist_ok=True) - # Verify that exists was called to check /var/lib/litellm/assets/logo.jpg - assets_logo_path = "/var/lib/litellm/assets/logo.jpg" - assert any(assets_logo_path in str(call) for call in exists_calls), \ - f"Should check if {assets_logo_path} exists" + # Verify that exists was called to check /tmp/litellm_assets/logo.jpg + tmp_logo_path = "/tmp/litellm_assets/logo.jpg" + assert any(tmp_logo_path in str(call) for call in exists_calls), \ + f"Should check if {tmp_logo_path} exists" # Verify FileResponse was called (with fallback logo) assert mock_file_response.called, "FileResponse should be called" @@ -3026,12 +2976,12 @@ def test_get_image_root_case_uses_current_dir(monkeypatch): # Call the function get_image() - # Verify makedirs was NOT called with /var/lib/litellm/assets (should not create it for root case) - var_lib_assets_calls = [ + # Verify makedirs was NOT called with /tmp/litellm_assets (should not create it for root case) + tmp_assets_calls = [ call for call in mock_makedirs.call_args_list - if "/var/lib/litellm/assets" in str(call) + if "/tmp/litellm_assets" in str(call) ] - assert len(var_lib_assets_calls) == 0, "Should not create /var/lib/litellm/assets for root case" + assert len(tmp_assets_calls) == 0, "Should not create /tmp/litellm_assets for root case" # Verify FileResponse was called assert mock_file_response.called, "FileResponse should be called" From 05dd247ff5423c72b3c29b398ddec5e5866f1fd6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 11:45:31 -0800 Subject: [PATCH 050/388] Fix UI disappearing for development instances --- docker/Dockerfile.non_root | 24 ++--- litellm/proxy/proxy_server.py | 59 ++++++------- tests/test_litellm/proxy/test_proxy_server.py | 88 +++++++++++++++---- 3 files changed, 110 insertions(+), 61 deletions(-) diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 7e9147a124e..af1bb5b2022 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -40,7 +40,7 @@ COPY . . ENV LITELLM_NON_ROOT=true # Build Admin UI using the upstream command order while keeping a single RUN layer -RUN mkdir -p /tmp/litellm_ui && \ +RUN mkdir -p /var/lib/litellm/ui && \ npm install -g npm@latest && npm cache clean --force && \ cd /app/ui/litellm-dashboard && \ if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \ @@ -49,10 +49,10 @@ RUN mkdir -p /tmp/litellm_ui && \ rm -f package-lock.json && \ npm install --legacy-peer-deps && \ npm run build && \ - cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/ && \ - mkdir -p /tmp/litellm_assets && \ - cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg && \ - ( cd /tmp/litellm_ui && \ + cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \ + mkdir -p /var/lib/litellm/assets && \ + cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \ + ( cd /var/lib/litellm/ui && \ for html_file in *.html; do \ if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \ folder_name="${html_file%.html}" && \ @@ -111,8 +111,8 @@ COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /ap COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf COPY --from=builder /app/schema.prisma /app/ COPY --from=builder /wheels/ /wheels/ -COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui -COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets +COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui +COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets COPY --from=builder /app/.cache /app/.cache COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras COPY --from=builder \ @@ -145,8 +145,8 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \ # Permissions, cleanup, and Prisma prep RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ - mkdir -p /nonexistent /.npm /tmp/litellm_assets /tmp/litellm_ui && \ - chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \ + mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \ + chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \ pip uninstall jwt -y || true && \ pip uninstall PyJWT -y || true && \ pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \ @@ -156,11 +156,11 @@ RUN chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \ LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \ [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \ LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \ - chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \ - chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \ + chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \ [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \ chmod -R g+rX $PRISMA_PATH && \ chmod -R g+rX /app/.cache && \ diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f754e52796f..a1be8153b29 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -946,20 +946,19 @@ try: # This prevents mutating the packaged UI directory (e.g. site-packages or the repo checkout) # and ensures extensionless routes like /ui/login work via /index.html. is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - runtime_ui_path = "/tmp/litellm_ui" - if _dir_has_content(runtime_ui_path): - if is_non_root: + # Only use runtime UI path in Docker/non-root environments + # In local development, use the packaged UI directly + if is_non_root: + # Use /var/lib/litellm/ui for Docker (more secure than /tmp) + runtime_ui_path = "/var/lib/litellm/ui" + + if _dir_has_content(runtime_ui_path): verbose_proxy_logger.info( f"Using pre-built UI for non-root Docker: {runtime_ui_path}" ) + ui_path = runtime_ui_path else: - verbose_proxy_logger.info( - f"Using cached runtime UI directory: {runtime_ui_path}" - ) - ui_path = runtime_ui_path - else: - if is_non_root: verbose_proxy_logger.error( f"UI not found at {runtime_ui_path}. Attempting to populate it from packaged UI." ) @@ -967,32 +966,32 @@ try: f"Path exists: {os.path.exists(runtime_ui_path)}, Has content: {_dir_has_content(runtime_ui_path)}" ) - try: - os.makedirs(runtime_ui_path, exist_ok=True) - if not _dir_has_content(runtime_ui_path) and _dir_has_content( - packaged_ui_path - ): - shutil.copytree( - packaged_ui_path, - runtime_ui_path, - dirs_exist_ok=True, - ) - except Exception as e: - if is_non_root: + try: + os.makedirs(runtime_ui_path, exist_ok=True) + if not _dir_has_content(runtime_ui_path) and _dir_has_content( + packaged_ui_path + ): + shutil.copytree( + packaged_ui_path, + runtime_ui_path, + dirs_exist_ok=True, + ) + except Exception as e: verbose_proxy_logger.exception( f"Failed to populate runtime UI directory {runtime_ui_path} from {packaged_ui_path}: {e}" ) - else: - if _dir_has_content(runtime_ui_path): - if is_non_root: + else: + if _dir_has_content(runtime_ui_path): verbose_proxy_logger.info( f"Using populated UI for non-root Docker: {runtime_ui_path}" ) - else: - verbose_proxy_logger.info( - f"Using populated runtime UI directory: {runtime_ui_path}" - ) - ui_path = runtime_ui_path + ui_path = runtime_ui_path + else: + # Local development: use packaged UI directly, no runtime copy needed + verbose_proxy_logger.info( + f"Using packaged UI directory for local development: {packaged_ui_path}" + ) + ui_path = packaged_ui_path # Only modify files if a custom server root path is set if server_root_path and server_root_path != "/": @@ -8885,7 +8884,7 @@ def get_image(): default_site_logo = os.path.join(current_dir, "logo.jpg") is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" - assets_dir = "/tmp/litellm_assets" if is_non_root else current_dir + assets_dir = "/var/lib/litellm/assets" if is_non_root else current_dir if is_non_root: os.makedirs(assets_dir, exist_ok=True) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index fd7036b9940..dc0dec3437f 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -273,6 +273,10 @@ def test_sso_key_generate_shows_deprecation_banner(client_no_auth, monkeypatch): def test_restructure_ui_html_files_handles_nested_routes(tmp_path): + """ + Test that _restructure_ui_html_files correctly restructures HTML files. + Note: This function is only called when is_non_root is True (ui_path != packaged_ui_path). + """ from litellm.proxy import proxy_server ui_root = tmp_path / "ui" @@ -306,7 +310,10 @@ def test_restructure_ui_html_files_handles_nested_routes(tmp_path): def test_ui_extensionless_route_requires_restructure(tmp_path): - """Regression for non-root fallback: /ui/login expects login/index.html.""" + """ + Regression for non-root fallback: /ui/login expects login/index.html. + Note: Restructuring only happens when is_non_root is True (ui_path != packaged_ui_path). + """ from litellm.proxy import proxy_server @@ -331,6 +338,49 @@ def test_ui_extensionless_route_requires_restructure(tmp_path): assert "login" in response.text +def test_restructure_only_happens_when_non_root(monkeypatch): + """ + Test that restructuring logic only executes when LITELLM_NON_ROOT is true. + When is_non_root is False, ui_path == packaged_ui_path, so restructuring is skipped. + """ + # Test Case 1: is_non_root is True - ui_path != packaged_ui_path, so restructuring should happen + monkeypatch.setenv("LITELLM_NON_ROOT", "true") + + runtime_ui_path = "/var/lib/litellm/ui" + packaged_ui_path = "/some/packaged/ui/path" + + # Simulate the logic from proxy_server.py + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + if is_non_root: + ui_path = runtime_ui_path + else: + ui_path = packaged_ui_path + + # This is the condition that determines if restructuring happens + should_restructure = ui_path != packaged_ui_path + + assert is_non_root is True + assert should_restructure is True + assert ui_path == runtime_ui_path + + # Test Case 2: is_non_root is False - ui_path == packaged_ui_path, so restructuring should NOT happen + monkeypatch.delenv("LITELLM_NON_ROOT", raising=False) + + # Simulate the logic from proxy_server.py + is_non_root = os.getenv("LITELLM_NON_ROOT", "").lower() == "true" + if is_non_root: + ui_path = runtime_ui_path + else: + ui_path = packaged_ui_path + + # This is the condition that determines if restructuring happens + should_restructure = ui_path != packaged_ui_path + + assert is_non_root is False + assert should_restructure is False + assert ui_path == packaged_ui_path + + @pytest.mark.asyncio async def test_initialize_scheduled_jobs_credentials(monkeypatch): """ @@ -2856,9 +2906,9 @@ def test_root_redirect_when_docs_url_not_root_and_redirect_url_set(monkeypatch): assert response.headers["location"] == test_redirect_url -def test_get_image_non_root_uses_tmp_assets_dir(monkeypatch): +def test_get_image_non_root_uses_var_lib_assets_dir(monkeypatch): """ - Test that get_image uses /tmp/litellm_assets when LITELLM_NON_ROOT is true. + Test that get_image uses /var/lib/litellm/assets when LITELLM_NON_ROOT is true. """ from unittest.mock import patch @@ -2887,14 +2937,14 @@ def test_get_image_non_root_uses_tmp_assets_dir(monkeypatch): # Call the function get_image() - # Verify makedirs was called with /tmp/litellm_assets - mock_makedirs.assert_called_once_with("/tmp/litellm_assets", exist_ok=True) + # Verify makedirs was called with /var/lib/litellm/assets + mock_makedirs.assert_called_once_with("/var/lib/litellm/assets", exist_ok=True) def test_get_image_non_root_fallback_to_default_logo(monkeypatch): """ Test that get_image falls back to default_site_logo when logo doesn't exist - in /tmp/litellm_assets for non-root case. + in /var/lib/litellm/assets for non-root case. """ from unittest.mock import patch @@ -2904,13 +2954,13 @@ def test_get_image_non_root_fallback_to_default_logo(monkeypatch): monkeypatch.setenv("LITELLM_NON_ROOT", "true") monkeypatch.delenv("UI_LOGO_PATH", raising=False) - # Track path.exists calls to verify it checks /tmp/litellm_assets/logo.jpg + # Track path.exists calls to verify it checks /var/lib/litellm/assets/logo.jpg exists_calls = [] def exists_side_effect(path): exists_calls.append(path) - # Return False for /tmp/litellm_assets/logo.jpg to trigger fallback - if "/tmp/litellm_assets/logo.jpg" in path: + # Return False for /var/lib/litellm/assets/logo.jpg to trigger fallback + if "/var/lib/litellm/assets/logo.jpg" in path: return False return True @@ -2933,13 +2983,13 @@ def test_get_image_non_root_fallback_to_default_logo(monkeypatch): # Call the function get_image() - # Verify makedirs was called with /tmp/litellm_assets - mock_makedirs.assert_called_once_with("/tmp/litellm_assets", exist_ok=True) + # Verify makedirs was called with /var/lib/litellm/assets + mock_makedirs.assert_called_once_with("/var/lib/litellm/assets", exist_ok=True) - # Verify that exists was called to check /tmp/litellm_assets/logo.jpg - tmp_logo_path = "/tmp/litellm_assets/logo.jpg" - assert any(tmp_logo_path in str(call) for call in exists_calls), \ - f"Should check if {tmp_logo_path} exists" + # Verify that exists was called to check /var/lib/litellm/assets/logo.jpg + assets_logo_path = "/var/lib/litellm/assets/logo.jpg" + assert any(assets_logo_path in str(call) for call in exists_calls), \ + f"Should check if {assets_logo_path} exists" # Verify FileResponse was called (with fallback logo) assert mock_file_response.called, "FileResponse should be called" @@ -2976,12 +3026,12 @@ def test_get_image_root_case_uses_current_dir(monkeypatch): # Call the function get_image() - # Verify makedirs was NOT called with /tmp/litellm_assets (should not create it for root case) - tmp_assets_calls = [ + # Verify makedirs was NOT called with /var/lib/litellm/assets (should not create it for root case) + var_lib_assets_calls = [ call for call in mock_makedirs.call_args_list - if "/tmp/litellm_assets" in str(call) + if "/var/lib/litellm/assets" in str(call) ] - assert len(tmp_assets_calls) == 0, "Should not create /tmp/litellm_assets for root case" + assert len(var_lib_assets_calls) == 0, "Should not create /var/lib/litellm/assets for root case" # Verify FileResponse was called assert mock_file_response.called, "FileResponse should be called" From 564b2b51cc904a5a38809292aa699d2e39c67d04 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 16:09:17 -0800 Subject: [PATCH 051/388] Fix for dev env --- litellm/proxy/proxy_server.py | 25 +++++++++++-------- tests/test_litellm/proxy/test_proxy_server.py | 25 ++++++++++--------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a1be8153b29..f56c0c2b07a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -992,7 +992,6 @@ try: f"Using packaged UI directory for local development: {packaged_ui_path}" ) ui_path = packaged_ui_path - # Only modify files if a custom server root path is set if server_root_path and server_root_path != "/": # Iterate through files in the UI directory @@ -1078,18 +1077,22 @@ try: continue # Handle HTML file restructuring - # Always restructure the directory we actually serve, but avoid mutating the packaged UI. + # Always restructure the directory we actually serve. # This is critical for extensionless routes like /ui/login (expects login/index.html). - if ui_path != packaged_ui_path: - try: - _restructure_ui_html_files(ui_path) - except PermissionError as e: - verbose_proxy_logger.exception( - f"Permission error while restructuring UI directory {ui_path}: {e}" - ) - else: + # In development, we restructure directly in _experimental/out. + # In non-root Docker, we restructure in /var/lib/litellm/ui. + try: + _restructure_ui_html_files(ui_path) verbose_proxy_logger.info( - f"Skipping runtime HTML restructuring for packaged UI directory: {ui_path}" + f"Restructured UI directory: {ui_path}" + ) + except PermissionError as e: + verbose_proxy_logger.exception( + f"Permission error while restructuring UI directory {ui_path}: {e}" + ) + except Exception as e: + verbose_proxy_logger.exception( + f"Error while restructuring UI directory {ui_path}: {e}" ) except Exception: diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index dc0dec3437f..5c7ece04513 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -275,7 +275,7 @@ def test_sso_key_generate_shows_deprecation_banner(client_no_auth, monkeypatch): def test_restructure_ui_html_files_handles_nested_routes(tmp_path): """ Test that _restructure_ui_html_files correctly restructures HTML files. - Note: This function is only called when is_non_root is True (ui_path != packaged_ui_path). + Note: This function is always called now, both in development and non-root Docker environments. """ from litellm.proxy import proxy_server @@ -312,7 +312,7 @@ def test_restructure_ui_html_files_handles_nested_routes(tmp_path): def test_ui_extensionless_route_requires_restructure(tmp_path): """ Regression for non-root fallback: /ui/login expects login/index.html. - Note: Restructuring only happens when is_non_root is True (ui_path != packaged_ui_path). + Note: Restructuring always happens now, both in development and non-root Docker environments. """ from litellm.proxy import proxy_server @@ -338,12 +338,13 @@ def test_ui_extensionless_route_requires_restructure(tmp_path): assert "login" in response.text -def test_restructure_only_happens_when_non_root(monkeypatch): +def test_restructure_always_happens(monkeypatch): """ - Test that restructuring logic only executes when LITELLM_NON_ROOT is true. - When is_non_root is False, ui_path == packaged_ui_path, so restructuring is skipped. + Test that restructuring logic always executes regardless of LITELLM_NON_ROOT setting. + In development (is_non_root=False), restructuring happens directly in _experimental/out. + In non-root Docker (is_non_root=True), restructuring happens in /var/lib/litellm/ui. """ - # Test Case 1: is_non_root is True - ui_path != packaged_ui_path, so restructuring should happen + # Test Case 1: is_non_root is True - restructuring happens in /var/lib/litellm/ui monkeypatch.setenv("LITELLM_NON_ROOT", "true") runtime_ui_path = "/var/lib/litellm/ui" @@ -356,14 +357,14 @@ def test_restructure_only_happens_when_non_root(monkeypatch): else: ui_path = packaged_ui_path - # This is the condition that determines if restructuring happens - should_restructure = ui_path != packaged_ui_path + # Restructuring always happens now, regardless of ui_path vs packaged_ui_path + should_restructure = True assert is_non_root is True assert should_restructure is True assert ui_path == runtime_ui_path - # Test Case 2: is_non_root is False - ui_path == packaged_ui_path, so restructuring should NOT happen + # Test Case 2: is_non_root is False - restructuring happens directly in packaged_ui_path monkeypatch.delenv("LITELLM_NON_ROOT", raising=False) # Simulate the logic from proxy_server.py @@ -373,11 +374,11 @@ def test_restructure_only_happens_when_non_root(monkeypatch): else: ui_path = packaged_ui_path - # This is the condition that determines if restructuring happens - should_restructure = ui_path != packaged_ui_path + # Restructuring always happens now, even when ui_path == packaged_ui_path + should_restructure = True assert is_non_root is False - assert should_restructure is False + assert should_restructure is True assert ui_path == packaged_ui_path From 4b38db1398ad23839b363367bd1cc7806136d2d9 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 17:17:45 -0800 Subject: [PATCH 052/388] Allow Organization Admins to See Organization Tab --- .../components/SidebarProvider.tsx | 13 +-- .../hooks/organizations/useOrganizations.ts | 13 +++ .../src/components/leftnav.test.tsx | 104 +++++++++++++++--- .../src/components/leftnav.tsx | 31 +++++- 4 files changed, 125 insertions(+), 36 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.tsx b/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.tsx index c522d4ce1e5..8b934e10779 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/components/SidebarProvider.tsx @@ -1,4 +1,3 @@ -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import Sidebar from "@/components/leftnav"; interface SidebarProviderProps { @@ -8,17 +7,7 @@ interface SidebarProviderProps { } const SidebarProvider = ({ setPage, defaultSelectedKey, sidebarCollapsed }: SidebarProviderProps) => { - const { accessToken, userRole } = useAuthorized(); - - return ( - - ); + return ; }; export default SidebarProvider; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts new file mode 100644 index 00000000000..ed194203d9a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts @@ -0,0 +1,13 @@ +import { useQuery, UseQueryResult } from "@tanstack/react-query"; +import { createQueryKeys } from "../common/queryKeysFactory"; +import { organizationListCall, Organization } from "@/components/networking"; + +const organizationKeys = createQueryKeys("organizations"); + +export const useOrganizations = (accessToken: string | null): UseQueryResult => { + return useQuery({ + queryKey: organizationKeys.list({}), + queryFn: async () => await organizationListCall(accessToken!), + enabled: Boolean(accessToken), + }); +}; diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 1512c8b9350..09109300dce 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -1,15 +1,8 @@ -import { act, fireEvent, render, waitFor } from "@testing-library/react"; +import { act, fireEvent, screen, waitFor } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; +import { renderWithProviders } from "../../tests/test-utils"; import Sidebar from "./leftnav"; -// Stub ResizeObserver used by antd in jsdom -class ResizeObserver { - observe() {} - unobserve() {} - disconnect() {} -} -(global as any).ResizeObserver = ResizeObserver; - vi.mock("../utils/roles", () => { return { all_admin_roles: ["admin"], @@ -19,17 +12,53 @@ vi.mock("../utils/roles", () => { }; }); +const { mockUseAuthorized, mockUseOrganizations } = vi.hoisted(() => { + const mockUseAuthorized = vi.fn(() => ({ + userId: "test-user-id", + accessToken: "test-access-token", + userRole: "admin", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + })); + + const mockUseOrganizations = vi.fn(() => ({ + data: [], + isLoading: false, + error: null, + })); + + return { mockUseAuthorized, mockUseOrganizations }; +}); + +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: mockUseAuthorized, +})); + +vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ + useOrganizations: mockUseOrganizations, +})); + +vi.mock("@/app/(dashboard)/hooks/uiConfig/useUIConfig", () => { + return { + useUIConfig: () => ({ + data: { admin_ui_disabled: false }, + isLoading: false, + }), + }; +}); + describe("Sidebar (leftnav)", () => { const defaultProps = { - accessToken: null as string | null, setPage: vi.fn(), - userRole: "admin", defaultSelectedKey: "api-keys", collapsed: false, }; it("renders all top-level (non-nested) tabs for admin", () => { - const { getByText } = render(); + renderWithProviders(); const topLevelLabels = [ "Virtual Keys", @@ -51,19 +80,19 @@ describe("Sidebar (leftnav)", () => { ]; topLevelLabels.forEach((label) => { - expect(getByText(label)).toBeInTheDocument(); + expect(screen.getByText(label)).toBeInTheDocument(); }); }); it("expands a nested tab to reveal its children (Tools > Search Tools)", async () => { - const { getByText, queryByText } = render(); + renderWithProviders(); - expect(queryByText("Search Tools")).not.toBeInTheDocument(); + expect(screen.queryByText("Search Tools")).not.toBeInTheDocument(); act(() => { - fireEvent.click(getByText("Tools")); + fireEvent.click(screen.getByText("Tools")); }); await waitFor(() => { - expect(getByText("Search Tools")).toBeInTheDocument(); + expect(screen.getByText("Search Tools")).toBeInTheDocument(); }); }); it("has no duplicate keys among all menu items and their children", () => { @@ -82,7 +111,7 @@ describe("Sidebar (leftnav)", () => { return allKeys; } - const { container } = render(); + const { container } = renderWithProviders(); const allRenderedKeys = getAllKeysFromMenu(container); const keySet = new Set(); @@ -95,4 +124,43 @@ describe("Sidebar (leftnav)", () => { } expect(duplicates).toHaveLength(0); }); + + it("should show Organizations tab for organization admins", () => { + mockUseAuthorized.mockReturnValueOnce({ + userId: "org-admin-user-id", + accessToken: "test-access-token", + userRole: "viewer", + token: "test-token", + userEmail: "orgadmin@example.com", + premiumUser: false, + disabledPersonalKeyCreation: false, + showSSOBanner: false, + }); + + mockUseOrganizations.mockReturnValueOnce({ + data: [ + { + organization_id: "org-1", + organization_name: "Test Organization", + spend: 0, + max_budget: null, + models: [], + tpm_limit: null, + rpm_limit: null, + members: [ + { + user_id: "org-admin-user-id", + user_role: "org_admin", + }, + ], + }, + ], + isLoading: false, + error: null, + } as any); + + renderWithProviders(); + + expect(screen.getByText("Organizations")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index ec000f7582e..6f716ccd66c 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -1,3 +1,5 @@ +import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { ApiOutlined, AppstoreOutlined, @@ -21,17 +23,17 @@ import { ToolOutlined, UserOutlined, } from "@ant-design/icons"; -import { Badge, ConfigProvider, Layout, Menu } from "antd"; import type { MenuProps } from "antd"; +import { Badge, ConfigProvider, Layout, Menu } from "antd"; +import { useMemo } from "react"; import { all_admin_roles, internalUserRoles, isAdminRole, rolesWithWriteAccess } from "../utils/roles"; +import type { Organization } from "./networking"; import UsageIndicator from "./usage_indicator"; const { Sider } = Layout; // Define the props type interface SidebarProps { - accessToken: string | null; setPage: (page: string) => void; - userRole: string; defaultSelectedKey: string; collapsed?: boolean; } @@ -53,7 +55,18 @@ interface MenuGroup { roles?: string[]; } -const Sidebar: React.FC = ({ accessToken, setPage, userRole, defaultSelectedKey, collapsed = false }) => { +const Sidebar: React.FC = ({ setPage, defaultSelectedKey, collapsed = false }) => { + const { userId, accessToken, userRole } = useAuthorized(); + const { data: organizations } = useOrganizations(accessToken); + + // Check if user is an org_admin + const isOrgAdmin = useMemo(() => { + if (!userId || !organizations) return false; + return organizations.some((org: Organization) => + org.members?.some((member) => member.user_id === userId && member.user_role === "org_admin"), + ); + }, [userId, organizations]); + // Navigate to page helper const navigateToPage = (page: string) => { const newSearchParams = new URLSearchParams(window.location.search); @@ -146,7 +159,7 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau Usage - ), + ), }, { key: "logs", @@ -302,7 +315,13 @@ const Sidebar: React.FC = ({ accessToken, setPage, userRole, defau // Filter items based on user role const filterItemsByRole = (items: MenuItem[]): MenuItem[] => { return items - .filter((item) => !item.roles || item.roles.includes(userRole)) + .filter((item) => { + // Special handling for organizations menu item - allow org_admins + if (item.key === "organizations") { + return !item.roles || item.roles.includes(userRole) || isOrgAdmin; + } + return !item.roles || item.roles.includes(userRole); + }) .map((item) => ({ ...item, children: item.children ? filterItemsByRole(item.children) : undefined, From 8b4c2da9ea712f38c6ae55af2c80eaacda654836 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 18:16:30 -0800 Subject: [PATCH 053/388] Resolve org alias on teams table --- .../src/components/OldTeams.test.tsx | 172 +++++++++++++++++- .../src/components/OldTeams.tsx | 18 +- 2 files changed, 179 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index f3b4ec82d53..76fc26a8847 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -1,10 +1,13 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import React from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { fetchAvailableModelsForTeamOrKey } from "./key_team_helpers/fetch_available_models_team_key"; import { fetchMCPAccessGroups, getGuardrailsList, teamCreateCall } from "./networking"; import OldTeams from "./OldTeams"; const mockTeamInfoView = vi.fn(); +const mockUseOrganizations = vi.fn(); vi.mock("./networking", () => ({ teamCreateCall: vi.fn(), @@ -57,6 +60,25 @@ vi.mock("@/components/team/team_info", () => ({ }, })); +vi.mock("@/app/(dashboard)/hooks/organizations/useOrganizations", () => ({ + useOrganizations: () => mockUseOrganizations(), +})); + +const createQueryClient = () => { + return new QueryClient({ + defaultOptions: { + queries: { + retry: false, + }, + }, + }); +}; + +const renderWithQueryClient = (component: React.ReactElement) => { + const queryClient = createQueryClient(); + return render({component}); +}; + describe("OldTeams - handleCreate organization handling", () => { beforeEach(() => { vi.clearAllMocks(); @@ -64,6 +86,7 @@ describe("OldTeams - handleCreate organization handling", () => { vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue([]); vi.mocked(fetchMCPAccessGroups).mockResolvedValue([]); vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [] }); + mockUseOrganizations.mockReturnValue({ data: null }); }); it("should not include organization_id when it's an empty string", async () => { @@ -274,7 +297,8 @@ describe("OldTeams - handleCreate organization handling", () => { }); it("should clear the delete modal when the cancel button is clicked", async () => { - render( + mockUseOrganizations.mockReturnValue({ data: [] }); + renderWithQueryClient( { describe("OldTeams - empty state", () => { beforeEach(() => { vi.clearAllMocks(); + mockUseOrganizations.mockReturnValue({ data: [] }); }); it("should display empty state message when teams array is empty", () => { - render( + renderWithQueryClient( { }); it("should display empty state message when teams is null", () => { - render( + renderWithQueryClient( { }); it("should not display empty state when teams array has items", () => { - render( + renderWithQueryClient( { vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue([]); vi.mocked(fetchMCPAccessGroups).mockResolvedValue([]); vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [] }); + mockUseOrganizations.mockReturnValue({ data: [] }); }); it("passes premiumUser flag to TeamInfoView", async () => { - render( + renderWithQueryClient( { describe("OldTeams - Default Team Settings tab visibility", () => { beforeEach(() => { vi.clearAllMocks(); + mockUseOrganizations.mockReturnValue({ data: [] }); }); it("should show Default Team Settings tab for Admin role", () => { - render( + renderWithQueryClient( { }); it("should show Default Team Settings tab for proxy_admin role", () => { - render( + renderWithQueryClient( { }); it("should not show Default Team Settings tab for proxy_admin_viewer role", () => { - render( + renderWithQueryClient( { }); it("should not show Default Team Settings tab for Admin Viewer role", () => { - render( + renderWithQueryClient( { beforeEach(() => { vi.clearAllMocks(); vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + mockUseOrganizations.mockReturnValue({ data: [] }); }); it("should not render all-proxy-models option in models select", async () => { vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); - render( + renderWithQueryClient( { expect(allProxyModelsOption).not.toBeInTheDocument(); }); }); + +describe("OldTeams - organization alias display", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockUseOrganizations.mockReturnValue({ data: [] }); + }); + + it("should display organization alias instead of organization id", () => { + const mockOrganizations = [ + { + organization_id: "org-123", + organization_alias: "Test Organization", + budget_id: "budget-1", + metadata: {}, + models: [], + spend: 0, + model_spend: {}, + created_at: new Date().toISOString(), + created_by: "user-1", + updated_at: new Date().toISOString(), + updated_by: "user-1", + litellm_budget_table: null, + teams: null, + users: null, + members: null, + }, + ]; + + mockUseOrganizations.mockReturnValue({ data: mockOrganizations }); + + renderWithQueryClient( + , + ); + + expect(screen.getByText("Test Organization")).toBeInTheDocument(); + expect(screen.queryByText("org-123")).not.toBeInTheDocument(); + }); + + it("should display organization id when alias is not found", () => { + mockUseOrganizations.mockReturnValue({ data: [] }); + + renderWithQueryClient( + , + ); + + expect(screen.getByText("org-unknown")).toBeInTheDocument(); + }); + + it("should display N/A when organization_id is null", () => { + mockUseOrganizations.mockReturnValue({ data: [] }); + + renderWithQueryClient( + , + ); + + expect(screen.getByText("N/A")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index 562d75c327a..77bbdb483da 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -1,3 +1,4 @@ +import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrganizations"; import AvailableTeamsPanel from "@/components/team/available_teams"; import TeamInfoView from "@/components/team/team_info"; import TeamSSOSettings from "@/components/TeamSSOSettings"; @@ -149,6 +150,18 @@ const getAdminOrganizations = ( return []; }; +const getOrganizationAlias = ( + organizationId: string | null | undefined, + organizations: Organization[] | null | undefined, +): string => { + if (!organizationId || !organizations) { + return organizationId || "N/A"; + } + + const organization = organizations.find((org) => org.organization_id === organizationId); + return organization?.organization_alias || organizationId; +}; + // @deprecated const Teams: React.FC = ({ teams, @@ -161,6 +174,7 @@ const Teams: React.FC = ({ premiumUser = false, }) => { console.log(`organizations: ${JSON.stringify(organizations)}`); + const { data: organizationsData } = useOrganizations(accessToken); const [lastRefreshed, setLastRefreshed] = useState(""); const [currentOrg, setCurrentOrg] = useState(null); const [currentOrgForCreateTeam, setCurrentOrgForCreateTeam] = useState(null); @@ -940,7 +954,9 @@ const Teams: React.FC = ({
- {team.organization_id} + + {getOrganizationAlias(team.organization_id, organizationsData || organizations)} + {perTeamInfo && From 42121ad13b6cf2b0b06f2a2f7e908158fc1c2fe8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 24 Dec 2025 09:25:36 +0530 Subject: [PATCH 054/388] Update minimax docs to be in proper format --- docs/my-website/docs/providers/minimax.md | 254 ++++++++++++++++ docs/my-website/docs/providers/minimax_tts.md | 281 ------------------ docs/my-website/docs/text_to_speech.md | 4 +- docs/my-website/sidebars.js | 1 - 4 files changed, 256 insertions(+), 284 deletions(-) delete mode 100644 docs/my-website/docs/providers/minimax_tts.md diff --git a/docs/my-website/docs/providers/minimax.md b/docs/my-website/docs/providers/minimax.md index 250c5159a3d..9505c26aade 100644 --- a/docs/my-website/docs/providers/minimax.md +++ b/docs/my-website/docs/providers/minimax.md @@ -382,4 +382,258 @@ response = litellm.completion( print(f"Cost: ${response._hidden_params.get('response_cost', 0)}") ``` +# MiniMax - Text-to-Speech +## Quick Start + +## **LiteLLM Python SDK Usage** + +### Basic Usage + +```python +from pathlib import Path +from litellm import speech +import os + +os.environ["MINIMAX_API_KEY"] = "your-api-key" + +speech_file_path = Path(__file__).parent / "speech.mp3" +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="The quick brown fox jumped over the lazy dogs", +) +response.stream_to_file(speech_file_path) +``` + +### Async Usage + +```python +from litellm import aspeech +from pathlib import Path +import os, asyncio + +os.environ["MINIMAX_API_KEY"] = "your-api-key" + +async def test_async_speech(): + speech_file_path = Path(__file__).parent / "speech.mp3" + response = await aspeech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="The quick brown fox jumped over the lazy dogs", + ) + response.stream_to_file(speech_file_path) + +asyncio.run(test_async_speech()) +``` + +### Voice Selection + +MiniMax supports many voices. LiteLLM provides OpenAI-compatible voice names that map to MiniMax voices: + +```python +from litellm import speech + +# OpenAI-compatible voice names +voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] + +for voice in voices: + response = speech( + model="minimax/speech-2.6-hd", + voice=voice, + input=f"This is the {voice} voice", + ) + response.stream_to_file(f"speech_{voice}.mp3") +``` + +You can also use MiniMax-native voice IDs directly: + +```python +response = speech( + model="minimax/speech-2.6-hd", + voice="male-qn-qingse", # MiniMax native voice ID + input="Using native MiniMax voice ID", +) +``` + +### Custom Parameters + +MiniMax TTS supports additional parameters for fine-tuning audio output: + +```python +from litellm import speech + +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="Custom audio parameters", + speed=1.5, # Speed: 0.5 to 2.0 + response_format="mp3", # Format: mp3, pcm, wav, flac + extra_body={ + "vol": 1.2, # Volume: 0.1 to 10 + "pitch": 2, # Pitch adjustment: -12 to 12 + "sample_rate": 32000, # 16000, 24000, or 32000 + "bitrate": 128000, # For MP3: 64000, 128000, 192000, 256000 + "channel": 1, # 1 for mono, 2 for stereo + } +) +response.stream_to_file("custom_speech.mp3") +``` + +### Response Formats + +```python +from litellm import speech + +# MP3 format (default) +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="MP3 format audio", + response_format="mp3", +) + +# PCM format +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="PCM format audio", + response_format="pcm", +) + +# WAV format +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="WAV format audio", + response_format="wav", +) + +# FLAC format +response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="FLAC format audio", + response_format="flac", +) +``` + +## **LiteLLM Proxy Usage** + +LiteLLM provides an OpenAI-compatible `/audio/speech` endpoint for MiniMax TTS. + +### Setup + +Add MiniMax to your proxy configuration: + +```yaml +model_list: + - model_name: tts + litellm_params: + model: minimax/speech-2.6-hd + api_key: os.environ/MINIMAX_API_KEY + + - model_name: tts-turbo + litellm_params: + model: minimax/speech-2.6-turbo + api_key: os.environ/MINIMAX_API_KEY +``` + +Start the proxy: + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### Making Requests + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts", + "input": "The quick brown fox jumped over the lazy dog.", + "voice": "alloy" + }' \ + --output speech.mp3 +``` + +With custom parameters: + +```bash +curl http://0.0.0.0:4000/v1/audio/speech \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "tts", + "input": "Custom parameters example.", + "voice": "nova", + "speed": 1.5, + "response_format": "mp3", + "extra_body": { + "vol": 1.2, + "pitch": 1, + "sample_rate": 32000 + } + }' \ + --output custom_speech.mp3 +``` + +## Voice Mappings + +LiteLLM maps OpenAI-compatible voice names to MiniMax voice IDs: + +| OpenAI Voice | MiniMax Voice ID | Description | +|--------------|------------------|-------------| +| alloy | male-qn-qingse | Male voice | +| echo | male-qn-jingying | Male voice | +| fable | female-shaonv | Female voice | +| onyx | male-qn-badao | Male voice | +| nova | female-yujie | Female voice | +| shimmer | female-tianmei | Female voice | + +You can also use any MiniMax-native voice ID directly by passing it as the `voice` parameter. + + +### Streaming (WebSocket) + +:::note +The current implementation uses MiniMax's HTTP endpoint. For WebSocket streaming support, please refer to MiniMax's official documentation at [https://platform.minimax.io/docs](https://platform.minimax.io/docs). +::: + +## Error Handling + +```python +from litellm import speech +import litellm + +try: + response = speech( + model="minimax/speech-2.6-hd", + voice="alloy", + input="Test input", + ) + response.stream_to_file("output.mp3") +except litellm.exceptions.BadRequestError as e: + print(f"Bad request: {e}") +except litellm.exceptions.AuthenticationError as e: + print(f"Authentication failed: {e}") +except Exception as e: + print(f"Error: {e}") +``` + +### Extra Body Parameters + +Pass these via `extra_body`: + +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| vol | float | Volume (0.1 to 10) | 1.0 | +| pitch | int | Pitch adjustment (-12 to 12) | 0 | +| sample_rate | int | Sample rate: 16000, 24000, 32000 | 32000 | +| bitrate | int | Bitrate for MP3: 64000, 128000, 192000, 256000 | 128000 | +| channel | int | Audio channels: 1 (mono) or 2 (stereo) | 1 | +| output_format | string | Output format: "hex" or "url" (url returns a URL valid for 24 hours) | hex | diff --git a/docs/my-website/docs/providers/minimax_tts.md b/docs/my-website/docs/providers/minimax_tts.md deleted file mode 100644 index 314d705413b..00000000000 --- a/docs/my-website/docs/providers/minimax_tts.md +++ /dev/null @@ -1,281 +0,0 @@ -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - -# MiniMax - Text-to-Speech - -## Overview - -MiniMax provides high-quality text-to-speech synthesis with support for 40+ languages and ultra-low latency. LiteLLM provides a unified OpenAI-compatible interface for MiniMax TTS. - -| Feature | Supported | Notes | -|---------|-----------|-------| -| Logging | ✅ | Works across all integrations | -| Fallbacks | ✅ | Works between supported models | -| Loadbalancing | ✅ | Works between supported models | -| Guardrails | ✅ | Applies to input text | -| Supported Models | speech-2.6-hd, speech-2.6-turbo, speech-02-hd, speech-02-turbo | | - -## Supported Models - -| Model | Description | -|-------|-------------| -| speech-2.6-hd | Ultra-low latency, intelligence parsing, and enhanced naturalness | -| speech-2.6-turbo | Faster, more affordable, ideal for agents | -| speech-02-hd | Superior rhythm and stability with outstanding replication similarity | -| speech-02-turbo | Superior rhythm and stability with enhanced multilingual capabilities | -| speech-01-hd | Previous generation HD model | -| speech-01-turbo | Previous generation turbo model | - -## Quick Start - -## **LiteLLM Python SDK Usage** - -### Basic Usage - -```python -from pathlib import Path -from litellm import speech -import os - -os.environ["MINIMAX_API_KEY"] = "your-api-key" - -speech_file_path = Path(__file__).parent / "speech.mp3" -response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="The quick brown fox jumped over the lazy dogs", -) -response.stream_to_file(speech_file_path) -``` - -### Async Usage - -```python -from litellm import aspeech -from pathlib import Path -import os, asyncio - -os.environ["MINIMAX_API_KEY"] = "your-api-key" - -async def test_async_speech(): - speech_file_path = Path(__file__).parent / "speech.mp3" - response = await aspeech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="The quick brown fox jumped over the lazy dogs", - ) - response.stream_to_file(speech_file_path) - -asyncio.run(test_async_speech()) -``` - -### Voice Selection - -MiniMax supports many voices. LiteLLM provides OpenAI-compatible voice names that map to MiniMax voices: - -```python -from litellm import speech - -# OpenAI-compatible voice names -voices = ["alloy", "echo", "fable", "onyx", "nova", "shimmer"] - -for voice in voices: - response = speech( - model="minimax/speech-2.6-hd", - voice=voice, - input=f"This is the {voice} voice", - ) - response.stream_to_file(f"speech_{voice}.mp3") -``` - -You can also use MiniMax-native voice IDs directly: - -```python -response = speech( - model="minimax/speech-2.6-hd", - voice="male-qn-qingse", # MiniMax native voice ID - input="Using native MiniMax voice ID", -) -``` - -### Custom Parameters - -MiniMax TTS supports additional parameters for fine-tuning audio output: - -```python -from litellm import speech - -response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="Custom audio parameters", - speed=1.5, # Speed: 0.5 to 2.0 - response_format="mp3", # Format: mp3, pcm, wav, flac - extra_body={ - "vol": 1.2, # Volume: 0.1 to 10 - "pitch": 2, # Pitch adjustment: -12 to 12 - "sample_rate": 32000, # 16000, 24000, or 32000 - "bitrate": 128000, # For MP3: 64000, 128000, 192000, 256000 - "channel": 1, # 1 for mono, 2 for stereo - } -) -response.stream_to_file("custom_speech.mp3") -``` - -### Response Formats - -```python -from litellm import speech - -# MP3 format (default) -response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="MP3 format audio", - response_format="mp3", -) - -# PCM format -response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="PCM format audio", - response_format="pcm", -) - -# WAV format -response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="WAV format audio", - response_format="wav", -) - -# FLAC format -response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="FLAC format audio", - response_format="flac", -) -``` - -## **LiteLLM Proxy Usage** - -LiteLLM provides an OpenAI-compatible `/audio/speech` endpoint for MiniMax TTS. - -### Setup - -Add MiniMax to your proxy configuration: - -```yaml -model_list: - - model_name: tts - litellm_params: - model: minimax/speech-2.6-hd - api_key: os.environ/MINIMAX_API_KEY - - - model_name: tts-turbo - litellm_params: - model: minimax/speech-2.6-turbo - api_key: os.environ/MINIMAX_API_KEY -``` - -Start the proxy: - -```bash -litellm --config /path/to/config.yaml - -# RUNNING on http://0.0.0.0:4000 -``` - -### Making Requests - -```bash -curl http://0.0.0.0:4000/v1/audio/speech \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "tts", - "input": "The quick brown fox jumped over the lazy dog.", - "voice": "alloy" - }' \ - --output speech.mp3 -``` - -With custom parameters: - -```bash -curl http://0.0.0.0:4000/v1/audio/speech \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ - "model": "tts", - "input": "Custom parameters example.", - "voice": "nova", - "speed": 1.5, - "response_format": "mp3", - "extra_body": { - "vol": 1.2, - "pitch": 1, - "sample_rate": 32000 - } - }' \ - --output custom_speech.mp3 -``` - -## Voice Mappings - -LiteLLM maps OpenAI-compatible voice names to MiniMax voice IDs: - -| OpenAI Voice | MiniMax Voice ID | Description | -|--------------|------------------|-------------| -| alloy | male-qn-qingse | Male voice | -| echo | male-qn-jingying | Male voice | -| fable | female-shaonv | Female voice | -| onyx | male-qn-badao | Male voice | -| nova | female-yujie | Female voice | -| shimmer | female-tianmei | Female voice | - -You can also use any MiniMax-native voice ID directly by passing it as the `voice` parameter. - - -### Streaming (WebSocket) - -:::note -The current implementation uses MiniMax's HTTP endpoint. For WebSocket streaming support, please refer to MiniMax's official documentation at [https://platform.minimax.io/docs](https://platform.minimax.io/docs). -::: - -## Error Handling - -```python -from litellm import speech -import litellm - -try: - response = speech( - model="minimax/speech-2.6-hd", - voice="alloy", - input="Test input", - ) - response.stream_to_file("output.mp3") -except litellm.exceptions.BadRequestError as e: - print(f"Bad request: {e}") -except litellm.exceptions.AuthenticationError as e: - print(f"Authentication failed: {e}") -except Exception as e: - print(f"Error: {e}") -``` - -### Extra Body Parameters - -Pass these via `extra_body`: - -| Parameter | Type | Description | Default | -|-----------|------|-------------|---------| -| vol | float | Volume (0.1 to 10) | 1.0 | -| pitch | int | Pitch adjustment (-12 to 12) | 0 | -| sample_rate | int | Sample rate: 16000, 24000, 32000 | 32000 | -| bitrate | int | Bitrate for MP3: 64000, 128000, 192000, 256000 | 128000 | -| channel | int | Audio channels: 1 (mono) or 2 (stereo) | 1 | -| output_format | string | Output format: "hex" or "url" (url returns a URL valid for 24 hours) | hex | diff --git a/docs/my-website/docs/text_to_speech.md b/docs/my-website/docs/text_to_speech.md index f5788630949..77d15ccb3a5 100644 --- a/docs/my-website/docs/text_to_speech.md +++ b/docs/my-website/docs/text_to_speech.md @@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem'; | Fallbacks | ✅ | Works between supported models | | Loadbalancing | ✅ | Works between supported models | | Guardrails | ✅ | Applies to input text (non-streaming only) | -| Supported Providers | OpenAI, Azure OpenAI, Vertex AI, AWS Polly, ElevenLabs , MiniMax| | +| Supported Providers | OpenAI, Azure OpenAI, Vertex AI, AWS Polly, ElevenLabs , MiniMax | ## **LiteLLM Python SDK Usage** ### Quick Start @@ -105,7 +105,7 @@ litellm --config /path/to/config.yaml | Vertex AI | [Usage](../docs/providers/vertex#text-to-speech-apis) | | Gemini | [Usage](#gemini-text-to-speech) | | ElevenLabs | [Usage](../docs/providers/elevenlabs#text-to-speech-tts) | -| MiniMax | [Usage](../docs/providers/minimax_tts) | +| MiniMax | [Usage](../docs/providers/minimax#minimax---text-to-speech) | ## `/audio/speech` to `/chat/completions` Bridge diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 948014a792b..c1d8387ec58 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -730,7 +730,6 @@ const sidebars = { "providers/milvus_vector_stores", "providers/mistral", "providers/minimax", - "providers/minimax_tts", "providers/moonshot", "providers/morph", "providers/nebius", From 40549bc361cb1a3fc587d0d7a30d1f3726e4e04a Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 23 Dec 2025 20:15:42 -0800 Subject: [PATCH 055/388] Deprecate useTeam in favor of react query --- .../app/(dashboard)/hooks/teams/useTeams.ts | 17 ++++ .../src/app/(dashboard)/hooks/useTeams.tsx | 4 + .../organization/organization_view.test.tsx | 78 +++++++++++++++++++ .../organization/organization_view.tsx | 9 ++- .../src/utils/teamUtils.test.ts | 31 +++++++- ui/litellm-dashboard/src/utils/teamUtils.ts | 24 +++++- 6 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts new file mode 100644 index 00000000000..8fb494539b0 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -0,0 +1,17 @@ +import { useQuery, UseQueryResult } from "@tanstack/react-query"; +import { Team } from "@/components/key_team_helpers/key_list"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { fetchTeams } from "@/app/(dashboard)/networking"; +import { createQueryKeys } from "@/app/(dashboard)/hooks/common/queryKeysFactory"; + +const teamKeys = createQueryKeys("teams"); + +export const useTeams = (): UseQueryResult => { + const { accessToken, userId: userID, userRole } = useAuthorized(); + + return useQuery({ + queryKey: teamKeys.list({}), + queryFn: async () => await fetchTeams(accessToken!, userID, userRole, null), + enabled: Boolean(accessToken), + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useTeams.tsx b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useTeams.tsx index 64cbf624f9c..0b3768505f5 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useTeams.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useTeams.tsx @@ -3,6 +3,10 @@ import { Team } from "@/components/key_team_helpers/key_list"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { fetchTeams } from "@/app/(dashboard)/networking"; +/** + * @deprecated This hook is deprecated. Use the react-query implementation from `@/app/(dashboard)/hooks/teams/useTeams` instead. + * This version will be removed in a future release. + */ const useTeams = () => { const [teams, setTeams] = useState([]); const { accessToken, userId: userID, userRole } = useAuthorized(); diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx index 5204efc9411..4320ad6c698 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx @@ -40,6 +40,24 @@ vi.mock("../mcp_server_management/MCPServerSelector", () => ({ __esModule: true, default: () => null, })); +const mockUseTeamsData = { + data: [ + { + team_id: "team_123", + team_alias: "Engineering Team", + }, + { + team_id: "team_456", + team_alias: "Marketing Team", + }, + ], +}; + +const mockUseTeams = vi.fn(() => mockUseTeamsData); + +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useTeams: () => mockUseTeams(), +})); const mockOrg = { organization_alias: "Acme Corp", @@ -103,3 +121,63 @@ test("should display empty state when organization has no members", async () => expect(screen.getByText("No members found")).toBeInTheDocument(); }); }); + +test("should display team aliases when teams are available", async () => { + const { organizationInfoCall } = await import("../networking"); + const orgWithTeams = { + ...mockOrg, + teams: [{ team_id: "team_123" }, { team_id: "team_456" }], + }; + (organizationInfoCall as unknown as ReturnType).mockResolvedValueOnce(orgWithTeams); + + render( + {}} + accessToken="test-token" + is_org_admin={false} + is_proxy_admin={false} + userModels={[]} + editOrg={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("Engineering Team")).toBeInTheDocument(); + expect(screen.getByText("Marketing Team")).toBeInTheDocument(); + }); +}); + +test("should display team ID as fallback when alias is not found", async () => { + const { organizationInfoCall } = await import("../networking"); + mockUseTeams.mockReturnValueOnce({ + data: [ + { + team_id: "team_123", + team_alias: "Engineering Team", + }, + ], + }); + + const orgWithUnknownTeam = { + ...mockOrg, + teams: [{ team_id: "team_999" }], + }; + (organizationInfoCall as unknown as ReturnType).mockResolvedValueOnce(orgWithUnknownTeam); + + render( + {}} + accessToken="test-token" + is_org_admin={false} + is_proxy_admin={false} + userModels={[]} + editOrg={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("team_999")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index 595987aadf5..91b2b4ce59b 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -23,7 +23,7 @@ import { } from "@tremor/react"; import { Button, Form, Input, Select } from "antd"; import { CheckIcon, CopyIcon } from "lucide-react"; -import React, { useEffect, useState } from "react"; +import React, { useEffect, useState, useMemo } from "react"; import UserSearchModal from "../common_components/user_search_modal"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; @@ -41,6 +41,8 @@ import ObjectPermissionsView from "../object_permissions_view"; import NumericalInput from "../shared/numerical_input"; import MemberModal from "../team/EditMembership"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { createTeamAliasMap } from "@/utils/teamUtils"; interface OrganizationInfoProps { organizationId: string; @@ -71,6 +73,9 @@ const OrganizationInfoView: React.FC = ({ const [copiedStates, setCopiedStates] = useState>({}); const [isOrgSaving, setIsOrgSaving] = useState(false); const canEditOrg = is_org_admin || is_proxy_admin; + const { data: teams } = useTeams(); + + const teamAliasMap = useMemo(() => createTeamAliasMap(teams), [teams]); const fetchOrgInfo = async () => { try { @@ -310,7 +315,7 @@ const OrganizationInfoView: React.FC = ({
{orgData.teams?.map((team, index) => ( - {team.team_id} + {teamAliasMap[team.team_id] || team.team_id} ))}
diff --git a/ui/litellm-dashboard/src/utils/teamUtils.test.ts b/ui/litellm-dashboard/src/utils/teamUtils.test.ts index 1151d532611..6c82da61854 100644 --- a/ui/litellm-dashboard/src/utils/teamUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/teamUtils.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { resolveTeamAliasFromTeamID } from "./teamUtils"; -import type { Team } from "@/components/networking"; +import { resolveTeamAliasFromTeamID, createTeamAliasMap } from "./teamUtils"; +import type { Team } from "@/components/key_team_helpers/key_list"; describe("resolveTeamAliasFromTeamID", () => { it("should return team alias when team is found", () => { @@ -35,3 +35,30 @@ describe("resolveTeamAliasFromTeamID", () => { expect(result).toBeNull(); }); }); + +describe("createTeamAliasMap", () => { + it("should create a map from team_id to team_alias", () => { + const teams = [ + { + team_id: "team1", + team_alias: "Team One", + }, + { + team_id: "team2", + team_alias: "Team Two", + }, + ] as unknown as Team[]; + + const result = createTeamAliasMap(teams); + expect(result).toEqual({ + team1: "Team One", + team2: "Team Two", + }); + }); + + it("should return empty object when teams is null or undefined", () => { + expect(createTeamAliasMap(null)).toEqual({}); + expect(createTeamAliasMap(undefined)).toEqual({}); + expect(createTeamAliasMap([])).toEqual({}); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/teamUtils.ts b/ui/litellm-dashboard/src/utils/teamUtils.ts index 1916e1c98aa..67100661a6f 100644 --- a/ui/litellm-dashboard/src/utils/teamUtils.ts +++ b/ui/litellm-dashboard/src/utils/teamUtils.ts @@ -1,5 +1,27 @@ -import { Team } from "@/components/networking"; +import { Team } from "@/components/key_team_helpers/key_list"; +/** + * Creates a map from team_id to team_alias for efficient lookups. + * @param teams - Array of Team objects + * @returns Record mapping team_id to team_alias + */ +export const createTeamAliasMap = (teams: Team[] | null | undefined): Record => { + if (!teams) return {}; + return teams.reduce( + (acc, team) => { + acc[team.team_id] = team.team_alias; + return acc; + }, + {} as Record, + ); +}; + +/** + * Resolves a team alias from a team ID. + * @param teamID - The team ID to look up + * @param teams - Array of Team objects + * @returns The team alias if found, null otherwise + */ export const resolveTeamAliasFromTeamID = (teamID: string, teams: Team[]): string | null => { const team = teams.find((team) => team.team_id === teamID); return team ? team.team_alias : null; From 61e1e42e972a035fd8e55c51daa45788f0e7c3ef Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Wed, 24 Dec 2025 14:12:17 +0900 Subject: [PATCH 056/388] fix: Datadog span kind fallback when parent_id missing --- .../integrations/datadog/datadog_llm_obs.py | 20 ++++--- .../datadog/test_datadog_llm_observability.py | 52 ++++++++++++------- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index 65ed8a795c0..6ffdbc0a005 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -217,8 +217,14 @@ class DataDogLLMObsLogger(CustomBatchLogger): error_info = self._assemble_error_info(standard_logging_payload) + metadata_parent_id: Optional[str] = None + if isinstance(metadata, dict): + metadata_parent_id = metadata.get("parent_id") + meta = Meta( - kind=self._get_datadog_span_kind(standard_logging_payload.get("call_type")), + kind=self._get_datadog_span_kind( + standard_logging_payload.get("call_type"), metadata_parent_id + ), input=input_meta, output=output_meta, metadata=self._get_dd_llm_obs_payload_metadata(standard_logging_payload), @@ -237,7 +243,7 @@ class DataDogLLMObsLogger(CustomBatchLogger): ) payload: LLMObsPayload = LLMObsPayload( - parent_id=metadata.get("parent_id", "undefined"), + parent_id=metadata_parent_id if metadata_parent_id else "undefined", trace_id=standard_logging_payload.get("trace_id", str(uuid.uuid4())), span_id=metadata.get("span_id", str(uuid.uuid4())), name=metadata.get("name", "litellm_llm_call"), @@ -367,14 +373,16 @@ class DataDogLLMObsLogger(CustomBatchLogger): return [] def _get_datadog_span_kind( - self, call_type: Optional[str] + self, call_type: Optional[str], parent_id: Optional[str] = None ) -> Literal["llm", "tool", "task", "embedding", "retrieval"]: """ Map liteLLM call_type to appropriate DataDog LLM Observability span kind. Available DataDog span kinds: "llm", "tool", "task", "embedding", "retrieval" + see: https://docs.datadoghq.com/ja/llm_observability/terms/ """ - if call_type is None: + # Non llm/workflow/agent kinds cannot be root spans, so fallback to "llm" when parent metadata is missing + if call_type is None or parent_id is None: return "llm" # Embedding operations @@ -392,6 +400,8 @@ class DataDogLLMObsLogger(CustomBatchLogger): CallTypes.generate_content_stream.value, CallTypes.agenerate_content_stream.value, CallTypes.anthropic_messages.value, + CallTypes.responses.value, + CallTypes.aresponses.value, ]: return "llm" @@ -417,8 +427,6 @@ class DataDogLLMObsLogger(CustomBatchLogger): CallTypes.aretrieve_batch.value, CallTypes.retrieve_fine_tuning_job.value, CallTypes.aretrieve_fine_tuning_job.value, - CallTypes.responses.value, - CallTypes.aresponses.value, CallTypes.alist_input_items.value, ]: return "retrieval" diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py index 464cb0026e5..48dec1fbc5a 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py @@ -257,41 +257,57 @@ class TestDataDogLLMObsLogger: logger = DataDogLLMObsLogger() # Test embedding operations - assert logger._get_datadog_span_kind(CallTypes.embedding.value) == "embedding" - assert logger._get_datadog_span_kind(CallTypes.aembedding.value) == "embedding" + assert logger._get_datadog_span_kind(CallTypes.embedding.value, "123") == "embedding" + assert logger._get_datadog_span_kind(CallTypes.aembedding.value, "123") == "embedding" # Test LLM completion operations - assert logger._get_datadog_span_kind(CallTypes.completion.value) == "llm" - assert logger._get_datadog_span_kind(CallTypes.acompletion.value) == "llm" - assert logger._get_datadog_span_kind(CallTypes.text_completion.value) == "llm" - assert logger._get_datadog_span_kind(CallTypes.generate_content.value) == "llm" + assert logger._get_datadog_span_kind(CallTypes.completion.value, None) == "llm" + assert logger._get_datadog_span_kind(CallTypes.acompletion.value, None) == "llm" + assert logger._get_datadog_span_kind(CallTypes.text_completion.value, None) == "llm" + assert logger._get_datadog_span_kind(CallTypes.generate_content.value, None) == "llm" assert ( - logger._get_datadog_span_kind(CallTypes.anthropic_messages.value) == "llm" + logger._get_datadog_span_kind(CallTypes.anthropic_messages.value, None) == "llm" ) + assert logger._get_datadog_span_kind(CallTypes.responses.value, None) == "llm" + assert logger._get_datadog_span_kind(CallTypes.aresponses.value, None) == "llm" # Test tool operations - assert logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value) == "tool" + assert logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value, "123") == "tool" # Test retrieval operations assert ( - logger._get_datadog_span_kind(CallTypes.get_assistants.value) == "retrieval" + logger._get_datadog_span_kind(CallTypes.get_assistants.value, "123") == "retrieval" ) assert ( - logger._get_datadog_span_kind(CallTypes.file_retrieve.value) == "retrieval" + logger._get_datadog_span_kind(CallTypes.file_retrieve.value, "123") == "retrieval" ) assert ( - logger._get_datadog_span_kind(CallTypes.retrieve_batch.value) == "retrieval" + logger._get_datadog_span_kind(CallTypes.retrieve_batch.value, "123") == "retrieval" ) # Test task operations - assert logger._get_datadog_span_kind(CallTypes.create_batch.value) == "task" - assert logger._get_datadog_span_kind(CallTypes.image_generation.value) == "task" - assert logger._get_datadog_span_kind(CallTypes.moderation.value) == "task" - assert logger._get_datadog_span_kind(CallTypes.transcription.value) == "task" + assert logger._get_datadog_span_kind(CallTypes.create_batch.value, "123") == "task" + assert logger._get_datadog_span_kind(CallTypes.image_generation.value, "123") == "task" + assert logger._get_datadog_span_kind(CallTypes.moderation.value, "123") == "task" + assert logger._get_datadog_span_kind(CallTypes.transcription.value, "123") == "task" # Test default fallback - assert logger._get_datadog_span_kind("unknown_call_type") == "llm" - assert logger._get_datadog_span_kind(None) == "llm" + assert logger._get_datadog_span_kind("unknown_call_type", None) == "llm" + assert logger._get_datadog_span_kind(None, None) == "llm" + + def test_datadog_span_kind_defaults_without_parent(self, mock_env_vars): + """Test that non-llm kinds fallback to llm when no parent span is provided""" + from litellm.types.utils import CallTypes + + with patch( + "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" + ), patch("asyncio.create_task"): + logger = DataDogLLMObsLogger() + + # Tool/task/retrieval span kinds should fallback to llm when parent_id missing + assert logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value, None) == "llm" + assert logger._get_datadog_span_kind(CallTypes.create_batch.value, None) == "llm" + assert logger._get_datadog_span_kind(CallTypes.get_assistants.value, None) == "llm" @pytest.mark.asyncio async def test_async_log_failure_event(self, mock_env_vars): @@ -796,7 +812,7 @@ class TestDataDogLLMObsLoggerToolCalls: from litellm.types.utils import CallTypes assert ( - logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value) == "tool" + logger._get_datadog_span_kind(CallTypes.call_mcp_tool.value, "123") == "tool" ) def test_tool_call_payload_creation(self, mock_env_vars): From 4c4c9cb35325d96d9276d7aa0dbb180752fa80eb Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 24 Dec 2025 12:16:24 +0530 Subject: [PATCH 057/388] Add generate content in llm route --- litellm/proxy/auth/route_checks.py | 20 +++++- .../proxy/auth/test_route_checks.py | 68 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 66973da7ee4..24f53b16bee 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -293,6 +293,9 @@ class RouteChecks: if route in LiteLLMRoutes.anthropic_routes.value: return True + + if route in LiteLLMRoutes.google_routes.value: + return True if RouteChecks.check_route_access( route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value @@ -315,13 +318,28 @@ class RouteChecks: ): return True + # Check for Google routes with placeholders like "/v1beta/models/{model_name}:generateContent" + for google_route in LiteLLMRoutes.google_routes.value: + if "{" in google_route: + if RouteChecks._route_matches_pattern( + route=route, pattern=google_route + ): + return True + + # Check for Anthropic routes with placeholders + for anthropic_route in LiteLLMRoutes.anthropic_routes.value: + if "{" in anthropic_route: + if RouteChecks._route_matches_pattern( + route=route, pattern=anthropic_route + ): + return True + if RouteChecks._is_azure_openai_route(route=route): return True for _llm_passthrough_route in LiteLLMRoutes.mapped_pass_through_routes.value: if _llm_passthrough_route in route: return True - return False @staticmethod diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py index b4b7ddbd9ea..ef7f2f3c30d 100644 --- a/tests/test_litellm/proxy/auth/test_route_checks.py +++ b/tests/test_litellm/proxy/auth/test_route_checks.py @@ -181,6 +181,74 @@ def test_virtual_key_llm_api_routes_allows_google_routes(route): assert result is True +@pytest.mark.parametrize( + "route", + [ + "/v1beta/models/google-gemini-2-5-pro-code-reviewer-k8s:generateContent", + "/v1beta/models/gemini-2.5-flash-exp:countTokens", + "/v1beta/models/custom-model-name-123:streamGenerateContent", + "/models/google-gemini-2-5-pro-code-reviewer-k8s:generateContent", + "/models/gemini-2.5-flash-exp:countTokens", + "/models/custom-model-name-123:streamGenerateContent", + ], +) +def test_google_routes_with_dynamic_model_names_recognized_as_llm_api_route(route): + """ + Test that Google routes with dynamic model names (including custom names) are recognized as LLM API routes. + + This test verifies the fix for the issue where routes like: + /v1beta/models/google-gemini-2-5-pro-code-reviewer-k8s:generateContent + were incorrectly classified as "custom admin only route" instead of LLM API routes. + + The fix adds pattern matching for Google routes with placeholders like {model_name}. + """ + + # Test that the route is recognized as an LLM API route + assert RouteChecks.is_llm_api_route(route) is True + + +def test_google_routes_with_dynamic_model_names_accessible_to_internal_users(): + """ + Test that internal users can access Google routes with dynamic model names. + + This ensures that routes like /v1beta/models/{model_name}:generateContent + are properly accessible to internal users and not blocked as admin-only routes. + """ + + # Create an internal user object + user_obj = LiteLLM_UserTable( + user_id="test_user", + user_email="test@example.com", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + + # Create an internal user API key auth + valid_token = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.INTERNAL_USER.value, + ) + + # Create a mock request + request = MagicMock(spec=Request) + request.query_params = {} + + # Test that calling Google route with dynamic model name does NOT raise an exception + try: + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=user_obj, + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/v1beta/models/google-gemini-2-5-pro-code-reviewer-k8s:generateContent", + request=request, + valid_token=valid_token, + request_data={"contents": [{"parts": [{"text": "test"}]}]}, + ) + # If no exception is raised, the test passes + except Exception as e: + pytest.fail( + f"Internal user should be able to access Google generateContent route. Got error: {str(e)}" + ) + + def test_virtual_key_allowed_routes_with_multiple_litellm_routes_member_names(): """Test that virtual key works with multiple LiteLLMRoutes member names in allowed_routes""" From 5abb677624b05ab64ffc9a985dde423ac3e77e10 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 24 Dec 2025 12:22:19 +0530 Subject: [PATCH 058/388] Remove double imports --- litellm/litellm_core_utils/default_encoding.py | 2 +- litellm/main.py | 14 +------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/default_encoding.py b/litellm/litellm_core_utils/default_encoding.py index d1f51e50720..41bfcbb63f4 100644 --- a/litellm/litellm_core_utils/default_encoding.py +++ b/litellm/litellm_core_utils/default_encoding.py @@ -31,7 +31,7 @@ for attempt in range(_max_retries): try: encoding = tiktoken.get_encoding("cl100k_base") break - except (FileExistsError, OSError) as e: + except (FileExistsError, OSError): if attempt == _max_retries - 1: # Last attempt, re-raise the exception raise diff --git a/litellm/main.py b/litellm/main.py index 6e069988726..a0f3461b45c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -110,30 +110,18 @@ from litellm.types.utils import ( RawRequestTypedDict, StreamingChoices, ) -from litellm.types.utils import ( - ModelResponseStream, - RawRequestTypedDict, - StreamingChoices, -) + from litellm.utils import ( - Choices, Choices, CustomStreamWrapper, EmbeddingResponse, Message, ModelResponse, - EmbeddingResponse, - Message, - ModelResponse, ProviderConfigManager, TextChoices, TextCompletionResponse, TextCompletionStreamWrapper, TranscriptionResponse, - TextChoices, - TextCompletionResponse, - TextCompletionStreamWrapper, - TranscriptionResponse, Usage, _get_model_info_helper, add_provider_specific_params_to_optional_params, From d02c580779100009f7e3f1a93cd92ff64fc92703 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 24 Dec 2025 13:51:58 +0530 Subject: [PATCH 059/388] feat(vertex-ai): add centralized get_vertex_base_url() helper for global location support - Add get_vertex_base_url() helper function to handle regional vs global URLs - Update _get_embedding_url() to support global location - Update _get_vertex_url() chat, image_generation, and count_tokens modes - Add comprehensive test suite with 38 tests covering all endpoint types - Tests verify both regional and global URL construction - Maintains 100% backward compatibility --- .../vertex_ai/agent_engine/transformation.py | 4 +- litellm/llms/vertex_ai/batches/handler.py | 4 +- litellm/llms/vertex_ai/common_utils.py | 65 ++- litellm/llms/vertex_ai/fine_tuning/handler.py | 20 +- .../vertex_gemini_transformation.py | 7 +- .../vertex_imagen_transformation.py | 4 +- .../vertex_gemini_transformation.py | 14 +- .../vertex_imagen_transformation.py | 3 +- litellm/llms/vertex_ai/ocr/transformation.py | 3 +- .../vertex_ai/rag_engine/transformation.py | 9 +- .../vector_stores/rag_api/transformation.py | 4 +- .../count_tokens/handler.py | 5 +- .../vertex_ai/vertex_model_garden/main.py | 5 +- .../llms/vertex_ai/videos/transformation.py | 5 +- .../test_vertex_global_url_support.py | 428 ++++++++++++++++++ 15 files changed, 518 insertions(+), 62 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/test_vertex_global_url_support.py diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py index 4c07e8455e3..42032079f94 100644 --- a/litellm/llms/vertex_ai/agent_engine/transformation.py +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( VertexAgentEngineResponseIterator, ) +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -130,8 +131,7 @@ class VertexAgentEngineConfig(BaseConfig, VertexBase): ) resource_path = f"projects/{vertex_project}/locations/{vertex_location}/reasoningEngines/{engine_id}" - # Build the base URL - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) # Always use :streamQuery endpoint for actual queries # The :query endpoint only supports session management methods diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index edae91ff9a3..12ce8b48aaf 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -8,6 +8,7 @@ from litellm.llms.custom_httpx.http_handler import ( _get_httpx_client, get_async_httpx_client, ) +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.types.llms.openai import CreateBatchRequest from litellm.types.llms.vertex_ai import ( @@ -128,7 +129,8 @@ class VertexAIBatchPrediction(VertexLLM): ) -> str: """Return the base url for the vertex garden models""" # POST https://LOCATION-aiplatform.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/batchPredictionJobs - return f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/batchPredictionJobs" + base_url = get_vertex_base_url(vertex_location) + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/batchPredictionJobs" def retrieve_batch( self, diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 03fa5b98928..7d84b7c9098 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -193,6 +193,18 @@ def get_vertex_base_model_name(model: str) -> str: return model +def get_vertex_base_url( + vertex_location: Optional[str], +) -> str: + """ + Get the base URL for Vertex AI API calls. + """ + if vertex_location == "global": + return "https://aiplatform.googleapis.com" + else: + return f"https://{vertex_location}-aiplatform.googleapis.com" + + def _get_embedding_url( model: str, vertex_project: Optional[str], @@ -212,10 +224,18 @@ def _get_embedding_url( # Strip routing prefixes (bge/, gemma/, etc.) for endpoint URL construction model = get_vertex_base_model_name(model=model) - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + # Get base URL (handles global vs regional) + base_url = get_vertex_base_url(vertex_location) + if model.isdigit(): # https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/endpoints/$ENDPOINT_ID:predict - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + # https://aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/global/endpoints/$ENDPOINT_ID:predict + url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + else: + # Regular model -> publisher model + # https://us-central1-aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/us-central1/publishers/google/models/{model}:predict + # https://aiplatform.googleapis.com/v1/projects/$PROJECT_ID/locations/global/publishers/google/models/{model}:predict + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" return url, endpoint @@ -236,26 +256,23 @@ def _get_vertex_url( if mode == "chat": ### SET RUNTIME ENDPOINT ### endpoint = "generateContent" + base_url = get_vertex_base_url(vertex_location) + if stream is True: endpoint = "streamGenerateContent" - if vertex_location == "global": - url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}?alt=sse" - else: - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}?alt=sse" - else: - if vertex_location == "global": - url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}" - else: - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" - + # if model is only numeric chars then it's a fine tuned gemini model # model = 4965075652664360960 - # send to this url: url = f"https://{vertex_location}-aiplatform.googleapis.com/{version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + # send to this url: url = f"{base_url}/{version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" if model.isdigit(): - # It's a fine-tuned Gemini model - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" - if stream is True: - url += "?alt=sse" + # It's a fine-tuned Gemini model - use endpoints/ path + url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + else: + # Regular model - use publishers/google/models/ path + url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + + if stream is True: + url += "?alt=sse" elif mode == "embedding": return _get_embedding_url( model=model, @@ -265,15 +282,17 @@ def _get_vertex_url( ) elif mode == "image_generation": endpoint = "predict" - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + base_url = get_vertex_base_url(vertex_location) if model.isdigit(): - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + # Numeric model -> custom endpoint + url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}:{endpoint}" + else: + # Regular model -> publisher model + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" elif mode == "count_tokens": endpoint = "countTokens" - if vertex_location == "global": - url = f"https://aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/global/publishers/google/models/{model}:{endpoint}" - else: - url = f"https://{vertex_location}-aiplatform.googleapis.com/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" + base_url = get_vertex_base_url(vertex_location) + url = f"{base_url}/{vertex_api_version}/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model}:{endpoint}" if not url or not endpoint: raise ValueError(f"Unable to get vertex url/endpoint for mode: {mode}") return url, endpoint diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 6372f8ea305..e2cd052fffd 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -8,6 +8,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.types.fine_tuning import OpenAIFineTuningHyperparameters from litellm.types.llms.openai import FineTuningJobCreate @@ -261,7 +262,8 @@ class VertexFineTuningAPI(VertexLLM): original_hyperparameters=original_hyperparameters or {}, ) - fine_tuning_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" + base_url = get_vertex_base_url(vertex_location) + fine_tuning_url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" if _is_async is True: return self.acreate_fine_tuning_job( # type: ignore fine_tuning_url=fine_tuning_url, @@ -329,19 +331,21 @@ class VertexFineTuningAPI(VertexLLM): "Content-Type": "application/json", } + base_url = get_vertex_base_url(vertex_location) + url = None if request_route == "/tuningJobs": - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs" elif "/tuningJobs/" in request_route and "cancel" in request_route: - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs{request_route}" + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/tuningJobs{request_route}" elif "generateContent" in request_route: - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "predict" in request_route: - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "/batchPredictionJobs" in request_route: - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "countTokens" in request_route: - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}{request_route}" elif "cachedContents" in request_route: _model = request_data.get("model") if _model is not None and "/publishers/google/models/" not in _model: @@ -349,7 +353,7 @@ class VertexFineTuningAPI(VertexLLM): f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}" ) - url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}" + url = f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}" else: raise ValueError(f"Unsupported Vertex AI request route: {request_route}") if self.async_handler is None: diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py index d575c5862e8..174d05cf7cf 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py @@ -10,6 +10,7 @@ from httpx._types import RequestFiles import litellm from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams @@ -143,11 +144,7 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM): if not vertex_project or not vertex_location: raise ValueError("vertex_project and vertex_location are required for Vertex AI") - # Handle global location differently (no region prefix in URL) - if vertex_location == "global": - base_url = "https://aiplatform.googleapis.com" - else: - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent" diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index ad650e38499..b61af6ffd3a 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -9,9 +9,9 @@ import httpx from httpx._types import RequestFiles import litellm - from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams @@ -136,7 +136,7 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): if api_base: base_url = api_base.rstrip("/") else: - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index 619bd006300..89ed9f1a8a5 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -7,13 +7,19 @@ import litellm from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails +from litellm.types.utils import ( + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -140,11 +146,7 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): if not vertex_project or not vertex_location: raise ValueError("vertex_project and vertex_location are required for Vertex AI") - # Handle global location differently (no region prefix in URL) - if vertex_location == "global": - base_url = "https://aiplatform.googleapis.com" - else: - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:generateContent" diff --git a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py index 33f416f9ca8..6f9e3874173 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py @@ -7,6 +7,7 @@ import litellm from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -140,7 +141,7 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): if not vertex_project or not vertex_location: raise ValueError("vertex_project and vertex_location are required for Vertex AI") - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" diff --git a/litellm/llms/vertex_ai/ocr/transformation.py b/litellm/llms/vertex_ai/ocr/transformation.py index f4482939851..849e332dae3 100644 --- a/litellm/llms/vertex_ai/ocr/transformation.py +++ b/litellm/llms/vertex_ai/ocr/transformation.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData from litellm.llms.mistral.ocr.transformation import MistralOCRConfig +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -104,7 +105,7 @@ class VertexAIOCRConfig(MistralOCRConfig): # Get API base URL if api_base is None: - api_base = f"https://{vertex_location}-aiplatform.googleapis.com" + api_base = get_vertex_base_url(vertex_location) # Ensure no trailing slash api_base = api_base.rstrip("/") diff --git a/litellm/llms/vertex_ai/rag_engine/transformation.py b/litellm/llms/vertex_ai/rag_engine/transformation.py index b601da1951a..7e70202fb75 100644 --- a/litellm/llms/vertex_ai/rag_engine/transformation.py +++ b/litellm/llms/vertex_ai/rag_engine/transformation.py @@ -8,6 +8,7 @@ from typing import Any, Dict, Optional from litellm._logging import verbose_logger from litellm.constants import DEFAULT_CHUNK_OVERLAP, DEFAULT_CHUNK_SIZE +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.rag import RAGChunkingStrategy @@ -37,8 +38,8 @@ class VertexAIRAGTransformation(VertexBase): Note: The REST endpoint for importRagFiles may not be publicly available. Vertex AI RAG Engine primarily uses gRPC-based SDK. """ - base_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1" - return f"{base_url}/projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{corpus_id}:importRagFiles" + base_url = get_vertex_base_url(vertex_location) + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/ragCorpora/{corpus_id}:importRagFiles" def get_retrieve_contexts_url( self, @@ -46,8 +47,8 @@ class VertexAIRAGTransformation(VertexBase): vertex_location: str, ) -> str: """Get the URL for retrieving contexts (search).""" - base_url = f"https://{vertex_location}-aiplatform.googleapis.com/v1" - return f"{base_url}/projects/{vertex_project}/locations/{vertex_location}:retrieveContexts" + base_url = get_vertex_base_url(vertex_location) + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}:retrieveContexts" def transform_chunking_strategy_to_vertex_format( self, diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 6f258bc04a6..08b93145e50 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.router import GenericLiteLLMParams from litellm.types.vector_stores import ( @@ -88,7 +89,8 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): return api_base.rstrip("/") # Vertex AI RAG API endpoint for retrieveContexts - return f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}" + base_url = get_vertex_base_url(vertex_location) + return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}" def transform_search_vector_store_request( self, diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index ae1a758bf20..3842159fd7b 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -8,6 +8,7 @@ their respective publisher-specific count-tokens endpoints. from typing import Any, Dict, Optional from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -65,10 +66,8 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): # Use custom api_base if provided, otherwise construct default if api_base: base_url = api_base - elif vertex_location == "global": - base_url = "https://aiplatform.googleapis.com" else: - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) # Construct the count-tokens endpoint # Format: /v1/projects/{project}/locations/{location}/publishers/{publisher}/models/count-tokens:rawPredict diff --git a/litellm/llms/vertex_ai/vertex_model_garden/main.py b/litellm/llms/vertex_ai/vertex_model_garden/main.py index fe7d0862e02..c37bb449ecf 100644 --- a/litellm/llms/vertex_ai/vertex_model_garden/main.py +++ b/litellm/llms/vertex_ai/vertex_model_garden/main.py @@ -20,6 +20,7 @@ from typing import Callable, Optional, Union import httpx # type: ignore +from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.utils import ModelResponse from ..common_utils import VertexAIError, get_vertex_base_model_name @@ -34,8 +35,8 @@ def create_vertex_url( api_base: Optional[str] = None, ) -> str: """Return the base url for the vertex garden models""" - # f"https://{self.endpoint.location}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{self.endpoint.location}" - return f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}" + base_url = get_vertex_base_url(vertex_location) + return f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}" class VertexAIModelGardenModels(VertexBase): diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index 8a542ae4ef0..66cd1437642 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -17,6 +17,7 @@ from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.vertex_ai.common_utils import ( _convert_vertex_datetime_to_openai_datetime, + get_vertex_base_url, ) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.router import GenericLiteLLMParams @@ -222,10 +223,8 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): # Construct the URL if api_base: base_url = api_base.rstrip("/") - elif vertex_location == "global": - base_url = "https://aiplatform.googleapis.com" else: - base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + base_url = get_vertex_base_url(vertex_location) url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}" diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_global_url_support.py b/tests/test_litellm/llms/vertex_ai/test_vertex_global_url_support.py new file mode 100644 index 00000000000..2c0178b3150 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_global_url_support.py @@ -0,0 +1,428 @@ +""" +Comprehensive tests for Vertex AI global URL support across all endpoints. + +This test suite ensures that all Vertex AI endpoints properly handle the 'global' location, +which uses a different URL format than regional endpoints. + +Regional: https://{region}-aiplatform.googleapis.com/... +Global: https://aiplatform.googleapis.com/... +""" + +from unittest.mock import patch + +import pytest + +from litellm.llms.vertex_ai.common_utils import ( + _get_embedding_url, + _get_vertex_url, + get_vertex_base_url, +) + + +class TestVertexBaseURL: + """Test the centralized get_vertex_base_url helper function.""" + + @pytest.mark.parametrize( + "vertex_location, expected_base_url", + [ + ("us-central1", "https://us-central1-aiplatform.googleapis.com"), + ("us-east1", "https://us-east1-aiplatform.googleapis.com"), + ("europe-west1", "https://europe-west1-aiplatform.googleapis.com"), + ("asia-northeast1", "https://asia-northeast1-aiplatform.googleapis.com"), + ("global", "https://aiplatform.googleapis.com"), + ], + ) + def test_get_vertex_base_url(self, vertex_location, expected_base_url): + """Test that get_vertex_base_url returns correct URL for all location types.""" + result = get_vertex_base_url(vertex_location) + assert result == expected_base_url + assert not result.endswith("/") # No trailing slash + + +class TestChatCompletionURLs: + """Test chat/completion endpoint URL construction with global location.""" + + @pytest.mark.parametrize( + "vertex_location, stream, expected_url_pattern", + [ + # Regional, non-streaming + ( + "us-central1", + False, + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent", + ), + # Regional, streaming + ( + "us-central1", + True, + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-1.5-pro:streamGenerateContent?alt=sse", + ), + # Global, non-streaming + ( + "global", + False, + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/publishers/google/models/gemini-1.5-pro:generateContent", + ), + # Global, streaming + ( + "global", + True, + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/publishers/google/models/gemini-1.5-pro:streamGenerateContent?alt=sse", + ), + ], + ) + def test_chat_url_construction( + self, vertex_location, stream, expected_url_pattern + ): + """Test that chat URLs are correctly constructed for regional and global locations.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, endpoint = _get_vertex_url( + mode="chat", + model="gemini-1.5-pro", + stream=stream, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version="v1", + ) + + assert url == expected_url_pattern + if stream: + assert endpoint == "streamGenerateContent" + assert "?alt=sse" in url + else: + assert endpoint == "generateContent" + assert "?alt=sse" not in url + + @pytest.mark.parametrize( + "vertex_location, stream", + [ + ("us-central1", False), + ("us-central1", True), + ("global", False), + ("global", True), + ], + ) + def test_finetuned_model_url_construction(self, vertex_location, stream): + """Test that fine-tuned models (numeric IDs) use endpoints/ path correctly.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, endpoint = _get_vertex_url( + mode="chat", + model="1234567890", # Numeric model ID + stream=stream, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version="v1", + ) + + # Should use endpoints/ path instead of publishers/google/models/ + assert "/endpoints/1234567890:" in url + assert "/publishers/google/models/" not in url + + # Check base URL is correct + if vertex_location == "global": + assert url.startswith("https://aiplatform.googleapis.com") + else: + assert url.startswith(f"https://{vertex_location}-aiplatform.googleapis.com") + + +class TestEmbeddingURLs: + """Test embedding endpoint URL construction with global location.""" + + @pytest.mark.parametrize( + "vertex_location, model, expected_url_pattern", + [ + # Regional, regular model + ( + "us-central1", + "text-embedding-004", + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/text-embedding-004:predict", + ), + # Global, regular model + ( + "global", + "text-embedding-004", + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/publishers/google/models/text-embedding-004:predict", + ), + # Regional, numeric endpoint + ( + "us-central1", + "1234567890", + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/endpoints/1234567890:predict", + ), + # Global, numeric endpoint + ( + "global", + "1234567890", + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/endpoints/1234567890:predict", + ), + ], + ) + def test_embedding_url_construction( + self, vertex_location, model, expected_url_pattern + ): + """Test that embedding URLs are correctly constructed for regional and global locations.""" + url, endpoint = _get_embedding_url( + model=model, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version="v1", + ) + + assert url == expected_url_pattern + assert endpoint == "predict" + + # Verify base URL format + if vertex_location == "global": + assert url.startswith("https://aiplatform.googleapis.com") + assert "-aiplatform.googleapis.com" not in url + else: + assert url.startswith(f"https://{vertex_location}-aiplatform.googleapis.com") + + @pytest.mark.parametrize( + "vertex_location", + ["us-central1", "europe-west1", "global"], + ) + def test_embedding_url_with_routing_prefix(self, vertex_location): + """Test that routing prefixes (bge/, gemma/, etc.) are stripped from URLs.""" + url, endpoint = _get_embedding_url( + model="bge/1234567890", # Model with routing prefix + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version="v1", + ) + + # Routing prefix should be stripped + assert "bge/" not in url + assert "/endpoints/1234567890:" in url + + +class TestCountTokensURLs: + """Test count_tokens endpoint URL construction with global location.""" + + @pytest.mark.parametrize( + "vertex_location, expected_url_pattern", + [ + ( + "us-central1", + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/gemini-1.5-pro:countTokens", + ), + ( + "global", + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/publishers/google/models/gemini-1.5-pro:countTokens", + ), + ], + ) + def test_count_tokens_url_construction(self, vertex_location, expected_url_pattern): + """Test that count_tokens URLs are correctly constructed for regional and global locations.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, endpoint = _get_vertex_url( + mode="count_tokens", + model="gemini-1.5-pro", + stream=None, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version="v1", + ) + + assert url == expected_url_pattern + assert endpoint == "countTokens" + + +class TestImageGenerationURLs: + """Test image_generation endpoint URL construction with global location.""" + + @pytest.mark.parametrize( + "vertex_location, model, expected_url_pattern", + [ + # Regional, regular model + ( + "us-central1", + "imagen-3.0-generate-001", + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/publishers/google/models/imagen-3.0-generate-001:predict", + ), + # Global, regular model + ( + "global", + "imagen-3.0-generate-001", + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/publishers/google/models/imagen-3.0-generate-001:predict", + ), + # Regional, numeric endpoint + ( + "us-central1", + "9876543210", + "https://us-central1-aiplatform.googleapis.com/v1/projects/test-project/locations/us-central1/endpoints/9876543210:predict", + ), + # Global, numeric endpoint + ( + "global", + "9876543210", + "https://aiplatform.googleapis.com/v1/projects/test-project/locations/global/endpoints/9876543210:predict", + ), + ], + ) + def test_image_generation_url_construction( + self, vertex_location, model, expected_url_pattern + ): + """Test that image_generation URLs are correctly constructed for regional and global locations.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, endpoint = _get_vertex_url( + mode="image_generation", + model=model, + stream=None, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version="v1", + ) + + assert url == expected_url_pattern + assert endpoint == "predict" + + +class TestAPIVersions: + """Test that both v1 and v1beta1 API versions work with global location.""" + + @pytest.mark.parametrize( + "api_version, vertex_location", + [ + ("v1", "us-central1"), + ("v1", "global"), + ("v1beta1", "us-central1"), + ("v1beta1", "global"), + ], + ) + def test_api_versions_in_urls(self, api_version, vertex_location): + """Test that API version is correctly included in URLs for all locations.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, _ = _get_vertex_url( + mode="chat", + model="gemini-1.5-pro", + stream=False, + vertex_project="test-project", + vertex_location=vertex_location, + vertex_api_version=api_version, + ) + + # API version should be in the URL + assert f"/{api_version}/" in url + + +class TestEdgeCases: + """Test edge cases and special scenarios.""" + + def test_global_location_no_region_prefix(self): + """Ensure global URLs never have a region prefix.""" + base_url = get_vertex_base_url("global") + assert base_url == "https://aiplatform.googleapis.com" + assert "global-aiplatform" not in base_url + assert "-aiplatform.googleapis.com" not in base_url + + @pytest.mark.parametrize( + "mode", + ["chat", "embedding", "count_tokens", "image_generation"], + ) + def test_all_modes_support_global(self, mode): + """Test that all URL modes support global location.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + if mode == "embedding": + url, _ = _get_embedding_url( + model="text-embedding-004", + vertex_project="test-project", + vertex_location="global", + vertex_api_version="v1", + ) + else: + url, _ = _get_vertex_url( + mode=mode, + model="gemini-1.5-pro", + stream=False, + vertex_project="test-project", + vertex_location="global", + vertex_api_version="v1", + ) + + # All URLs should use global format + assert url.startswith("https://aiplatform.googleapis.com") + assert "/locations/global/" in url + + def test_location_in_path_matches_parameter(self): + """Ensure the location in the URL path matches the vertex_location parameter.""" + test_locations = ["us-central1", "europe-west1", "global"] + + for location in test_locations: + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, _ = _get_vertex_url( + mode="chat", + model="gemini-1.5-pro", + stream=False, + vertex_project="test-project", + vertex_location=location, + vertex_api_version="v1", + ) + + # Location should appear in the path + assert f"/locations/{location}/" in url + + +class TestBackwardCompatibility: + """Ensure changes don't break existing functionality.""" + + def test_regional_urls_unchanged(self): + """Test that regional URL construction hasn't changed.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, _ = _get_vertex_url( + mode="chat", + model="gemini-1.5-pro", + stream=False, + vertex_project="my-project", + vertex_location="us-central1", + vertex_api_version="v1", + ) + + # Should match the traditional regional format + assert ( + url + == "https://us-central1-aiplatform.googleapis.com/v1/projects/my-project/locations/us-central1/publishers/google/models/gemini-1.5-pro:generateContent" + ) + + def test_streaming_urls_unchanged(self): + """Test that streaming URL construction hasn't changed.""" + with patch( + "litellm.VertexGeminiConfig.get_model_for_vertex_ai_url", + side_effect=lambda model: model, + ): + url, _ = _get_vertex_url( + mode="chat", + model="gemini-1.5-pro", + stream=True, + vertex_project="my-project", + vertex_location="us-central1", + vertex_api_version="v1", + ) + + # Should include streaming endpoint and alt=sse + assert ":streamGenerateContent?alt=sse" in url + From d88bc130062358fcfc46d8415f2cd9a429986688 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 24 Dec 2025 15:15:20 +0530 Subject: [PATCH 060/388] Add support for image generation via azure ad token --- litellm/images/main.py | 52 ++++++++++-- .../test_azure_image_generation_init.py | 79 ++++++++++++++++++- 2 files changed, 125 insertions(+), 6 deletions(-) diff --git a/litellm/images/main.py b/litellm/images/main.py index 03c0e36ad93..cf588cbcf0f 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -2,7 +2,18 @@ import asyncio import contextvars import importlib from functools import partial -from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Literal, Optional, Union, cast, overload +from typing import ( + TYPE_CHECKING, + Any, + Coroutine, + Dict, + List, + Literal, + Optional, + Union, + cast, + overload, +) if TYPE_CHECKING: from litellm.images.utils import ImageEditRequestUtils @@ -10,7 +21,7 @@ if TYPE_CHECKING: import httpx import litellm -from litellm.utils import exception_type, get_litellm_params + # client is imported from litellm as it's a decorator from litellm import client from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL @@ -23,6 +34,7 @@ from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.custom_llm import CustomLLM +from litellm.utils import exception_type, get_litellm_params #################### Initialize provider clients #################### llm_http_handler: BaseLLMHTTPHandler = BaseLLMHTTPHandler() @@ -32,8 +44,8 @@ from litellm.main import ( azure_chat_completions, base_llm_aiohttp_handler, base_llm_http_handler, - bedrock_image_generation, bedrock_image_edit, + bedrock_image_generation, openai_chat_completions, openai_image_variations, ) @@ -330,11 +342,36 @@ def image_generation( # noqa: PLR0915 azure_ad_token = optional_params.pop( "azure_ad_token", None ) or get_secret_str("AZURE_AD_TOKEN") + + # Create azure_ad_token_provider from tenant_id, client_id, client_secret if not already provided + if azure_ad_token_provider is None: + from litellm.llms.azure.common_utils import ( + get_azure_ad_token_from_entra_id, + ) + + # Extract Azure AD credentials from litellm_params + tenant_id = litellm_params_dict.get("tenant_id") + client_id = litellm_params_dict.get("client_id") + client_secret = litellm_params_dict.get("client_secret") + azure_scope = litellm_params_dict.get("azure_scope") or "https://cognitiveservices.azure.com/.default" + + # Create token provider if credentials are available + if tenant_id and client_id and client_secret: + azure_ad_token_provider = get_azure_ad_token_from_entra_id( + tenant_id=tenant_id, + client_id=client_id, + client_secret=client_secret, + scope=azure_scope, + ) default_headers = { "Content-Type": "application/json", - "api-key": api_key, } + # Only add api-key header if api_key is not None + # Azure AD authentication will use Authorization header instead + if api_key is not None: + default_headers["api-key"] = api_key + for k, v in default_headers.items(): if k not in headers: headers[k] = v @@ -399,8 +436,12 @@ def image_generation( # noqa: PLR0915 default_headers = { "Content-Type": "application/json", - "api-key": api_key, } + # Only add api-key header if api_key is not None + # Azure AD authentication will use Authorization header instead + if api_key is not None: + default_headers["api-key"] = api_key + for k, v in default_headers.items(): if k not in headers: headers[k] = v @@ -983,6 +1024,7 @@ def __getattr__(name: str) -> Any: if name == "ImageEditRequestUtils": # Lazy load ImageEditRequestUtils to avoid heavy import from images.utils at module load time from .utils import ImageEditRequestUtils as _ImageEditRequestUtils + # Cache it in the module's __dict__ for subsequent accesses module = importlib.import_module(__name__) module.__dict__["ImageEditRequestUtils"] = _ImageEditRequestUtils diff --git a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py index f31001ebd36..998510efcd9 100644 --- a/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py +++ b/tests/test_litellm/llms/azure/image_generation/test_azure_image_generation_init.py @@ -3,7 +3,7 @@ import os import sys import traceback from typing import Callable, Optional -from unittest.mock import MagicMock, patch +from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -87,3 +87,80 @@ def test_azure_image_generation_flattens_extra_body(): assert data["custom_param"] == "test_value" assert data["n"] == 1 assert data["size"] == "1024x1024" + + +def test_azure_image_generation_creates_token_provider_from_credentials(): + """ + Test that azure_ad_token_provider is created from tenant_id, client_id, client_secret. + + This test verifies the fix in images/main.py where we now create the + azure_ad_token_provider from credentials in litellm_params if it's not already provided. + """ + # Simulate the fix in images/main.py + litellm_params_dict = { + "tenant_id": "test-tenant-id", + "client_id": "test-client-id", + "client_secret": "test-client-secret", + "azure_scope": None, + } + + azure_ad_token_provider = None + + # This is the logic we added in images/main.py + if azure_ad_token_provider is None: + tenant_id = litellm_params_dict.get("tenant_id") + client_id = litellm_params_dict.get("client_id") + client_secret = litellm_params_dict.get("client_secret") + azure_scope = litellm_params_dict.get("azure_scope") or "https://cognitiveservices.azure.com/.default" + + # Verify the credentials are extracted correctly + assert tenant_id == "test-tenant-id" + assert client_id == "test-client-id" + assert client_secret == "test-client-secret" + assert azure_scope == "https://cognitiveservices.azure.com/.default" + + # Verify the condition to create token provider is met + assert tenant_id and client_id and client_secret, "Credentials should be present to create token provider" + + +def test_azure_image_generation_headers_without_api_key(): + """ + Test that when api_key is None, the api-key header is not added to headers. + + This prevents the httpx TypeError: "Header value must be str or bytes, not " + that was occurring when api_key was None and being set in headers. + + This is a unit test for the fix in images/main.py where we now check: + if api_key is not None: + default_headers["api-key"] = api_key + """ + from litellm.images.main import image_generation + + # Test the header building logic directly + api_key = None + + default_headers = { + "Content-Type": "application/json", + } + + # This is the fix: only add api-key if it's not None + if api_key is not None: + default_headers["api-key"] = api_key + + # Verify api-key is not in headers when api_key is None + assert "api-key" not in default_headers + + # Verify Content-Type is still there + assert default_headers["Content-Type"] == "application/json" + + # Test with a valid api_key + api_key = "valid-key-123" + default_headers_with_key = { + "Content-Type": "application/json", + } + if api_key is not None: + default_headers_with_key["api-key"] = api_key + + # Verify api-key is added when api_key is valid + assert "api-key" in default_headers_with_key + assert default_headers_with_key["api-key"] == "valid-key-123" From 825503fa1b7b3b10ddf6a59f93f96d5dca80b200 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 10:13:55 -0800 Subject: [PATCH 061/388] Refactor react-query hooks --- .../src/app/(dashboard)/hooks/agents/useAgents.ts | 4 +++- .../hooks/credentials/useCredentials.ts | 4 +++- .../(dashboard)/hooks/customers/useCustomers.ts | 7 ++++--- .../hooks/mcpServers/useMCPAccessGroups.ts | 7 ++++--- .../(dashboard)/hooks/mcpServers/useMCPServers.ts | 4 +++- .../src/app/(dashboard)/hooks/models/useModels.ts | 14 ++++++++------ .../hooks/organizations/useOrganizations.ts | 7 +++++-- .../src/app/(dashboard)/hooks/teams/useTeams.ts | 4 ++-- .../app/(dashboard)/hooks/useAuthorized.test.ts | 3 +++ .../ModelsAndEndpointsView.test.tsx | 13 ------------- .../ModelsAndEndpointsView.tsx | 8 ++------ .../UsagePage/components/UsagePageView.tsx | 4 ++-- ui/litellm-dashboard/src/components/leftnav.tsx | 2 +- .../mcp_server_management/MCPServerSelector.tsx | 4 ++-- .../MCPToolPermissions.test.tsx | 4 +++- .../src/components/mcp_tools/mcp_servers.test.tsx | 5 +++-- .../src/components/model_add/credentials.tsx | 2 +- ui/litellm-dashboard/tests/setupTests.ts | 14 ++++++++++++++ 18 files changed, 63 insertions(+), 47 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/agents/useAgents.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/agents/useAgents.ts index f2b7e76777d..d30eb345a0b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/agents/useAgents.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/agents/useAgents.ts @@ -3,10 +3,12 @@ import { AgentsResponse } from "@/components/agents/types"; import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { all_admin_roles } from "@/utils/roles"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; const agentsKeys = createQueryKeys("agents"); -export const useAgents = (accessToken: string | null, userRole: string | null) => { +export const useAgents = () => { + const { accessToken, userRole } = useAuthorized(); return useQuery({ queryKey: agentsKeys.list({}), queryFn: async () => await getAgentsList(accessToken!), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useCredentials.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useCredentials.ts index aa0a6c2c9fb..e3266de4fbc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useCredentials.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/credentials/useCredentials.ts @@ -1,10 +1,12 @@ import { credentialListCall, CredentialsResponse } from "@/components/networking"; import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; const credentialsKeys = createQueryKeys("credentials"); -export const useCredentials = (accessToken: string | null) => { +export const useCredentials = () => { + const { accessToken } = useAuthorized(); return useQuery({ queryKey: credentialsKeys.list({}), queryFn: async () => await credentialListCall(accessToken!), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts index 10cbedc04d3..d9f3e7cbb36 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/customers/useCustomers.ts @@ -2,7 +2,7 @@ import { allEndUsersCall } from "@/components/networking"; import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { all_admin_roles } from "@/utils/roles"; - +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; const customersKeys = createQueryKeys("customers"); export interface Customer { @@ -32,10 +32,11 @@ export interface Customer { export type CustomersResponse = Customer[]; -export const useCustomers = (accessToken: string | null, userRole: string | null) => { +export const useCustomers = () => { + const { accessToken, userRole } = useAuthorized(); return useQuery({ queryKey: customersKeys.list({}), queryFn: async () => await allEndUsersCall(accessToken!), - enabled: Boolean(accessToken) && all_admin_roles.includes(userRole || ""), + enabled: Boolean(accessToken) && all_admin_roles.includes(userRole!), }); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.ts index eeeb76bb742..0e88b62b0f3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPAccessGroups.ts @@ -1,13 +1,14 @@ import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { fetchMCPAccessGroups } from "@/components/networking"; - +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; const mcpAccessGroupsKeys = createQueryKeys("mcpAccessGroups"); -export const useMCPAccessGroups = (accessToken: string | null) => { +export const useMCPAccessGroups = () => { + const { accessToken } = useAuthorized(); return useQuery({ queryKey: mcpAccessGroupsKeys.list({}), queryFn: async () => await fetchMCPAccessGroups(accessToken!), - enabled: !!accessToken, + enabled: Boolean(accessToken), }); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts index 02e471d8e5f..8746baae148 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/mcpServers/useMCPServers.ts @@ -2,10 +2,12 @@ import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { fetchMCPServers } from "@/components/networking"; import { MCPServer } from "@/components/mcp_tools/types"; +import useAuthorized from "../useAuthorized"; const mcpServersKeys = createQueryKeys("mcpServers"); -export const useMCPServers = (accessToken: string | null) => { +export const useMCPServers = () => { + const { accessToken } = useAuthorized(); return useQuery({ queryKey: mcpServersKeys.list({}), queryFn: async () => await fetchMCPServers(accessToken!), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index aef05b1af2a..9c7ddf18f54 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -1,24 +1,26 @@ import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { modelInfoCall, modelHubCall } from "@/components/networking"; - +import useAuthorized from "../useAuthorized"; const modelKeys = createQueryKeys("models"); const modelHubKeys = createQueryKeys("modelHub"); -export const useModelsInfo = (accessToken: string | null, userID: string | null, userRole: string | null) => { +export const useModelsInfo = () => { + const { accessToken, userId, userRole } = useAuthorized(); return useQuery({ queryKey: modelKeys.list({ filters: { - ...(userID && { userID }), + ...(userId && { userId }), ...(userRole && { userRole }), }, }), - queryFn: async () => await modelInfoCall(accessToken!, userID!, userRole!), - enabled: Boolean(accessToken && userID && userRole), + queryFn: async () => await modelInfoCall(accessToken!, userId!, userRole!), + enabled: Boolean(accessToken && userId && userRole), }); }; -export const useModelHub = (accessToken: string | null) => { +export const useModelHub = () => { + const { accessToken } = useAuthorized(); return useQuery({ queryKey: modelHubKeys.list({}), queryFn: async () => await modelHubCall(accessToken!), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts index ed194203d9a..57c9c057652 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/organizations/useOrganizations.ts @@ -1,13 +1,16 @@ import { useQuery, UseQueryResult } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; import { organizationListCall, Organization } from "@/components/networking"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; const organizationKeys = createQueryKeys("organizations"); -export const useOrganizations = (accessToken: string | null): UseQueryResult => { +export const useOrganizations = (): UseQueryResult => { + const { accessToken } = useAuthorized(); + const { userId, userRole } = useAuthorized(); return useQuery({ queryKey: organizationKeys.list({}), queryFn: async () => await organizationListCall(accessToken!), - enabled: Boolean(accessToken), + enabled: Boolean(accessToken && userId && userRole), }); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts index 8fb494539b0..5d2008a4d29 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -7,11 +7,11 @@ import { createQueryKeys } from "@/app/(dashboard)/hooks/common/queryKeysFactory const teamKeys = createQueryKeys("teams"); export const useTeams = (): UseQueryResult => { - const { accessToken, userId: userID, userRole } = useAuthorized(); + const { accessToken, userId, userRole } = useAuthorized(); return useQuery({ queryKey: teamKeys.list({}), - queryFn: async () => await fetchTeams(accessToken!, userID, userRole, null), + queryFn: async () => await fetchTeams(accessToken!, userId, userRole, null), enabled: Boolean(accessToken), }); }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts index 26684619378..3da27d3ff9b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/useAuthorized.test.ts @@ -5,6 +5,9 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import useAuthorized from "./useAuthorized"; +// Unmock useAuthorized to test the actual implementation +vi.unmock("@/app/(dashboard)/hooks/useAuthorized"); + const { replaceMock, clearTokenCookiesMock, getProxyBaseUrlMock, getUiConfigMock } = vi.hoisted(() => ({ replaceMock: vi.fn(), clearTokenCookiesMock: vi.fn(), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx index 428f52dd98c..8dc7d1ff3d6 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.test.tsx @@ -37,19 +37,6 @@ vi.mock("@/app/(dashboard)/models-and-endpoints/components/ModelAnalyticsTab/Mod default: () => null, })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ - token: "123", - accessToken: "123", - userId: "user-1", - userEmail: "user@example.com", - userRole: "Admin", - premiumUser: false, - disabledPersonalKeyCreation: null, - showSSOBanner: false, - }), -})); - vi.mock("@/app/(dashboard)/hooks/useTeams", () => ({ default: () => ({ teams: [], diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index 4b71554ce22..969c7bafa6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -152,12 +152,8 @@ const ModelsAndEndpointsView: React.FC = ({ const [selectedTabIndex, setSelectedTabIndex] = useState(0); const queryClient = useQueryClient(); - const { - data: modelDataResponse, - isLoading: isLoadingModels, - refetch: refetchModels, - } = useModelsInfo(accessToken, userID, userRole); - const { data: credentialsResponse } = useCredentials(accessToken); + const { data: modelDataResponse, isLoading: isLoadingModels, refetch: refetchModels } = useModelsInfo(); + const { data: credentialsResponse } = useCredentials(); const credentialsList = credentialsResponse?.credentials || []; const { data: uiSettings } = useUISettings(accessToken || ""); diff --git a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx index 9766983c36a..920955138b9 100644 --- a/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/components/UsagePage/components/UsagePageView.tsx @@ -80,8 +80,8 @@ const UsagePage: React.FC = ({ teams, organizations }) => { }); const [allTags, setAllTags] = useState([]); - const { data: customers = [] } = useCustomers(accessToken, userRole); - const { data: agentsResponse } = useAgents(accessToken, userRole); + const { data: customers = [] } = useCustomers(); + const { data: agentsResponse } = useAgents(); const [modelViewType, setModelViewType] = useState<"groups" | "individual">("groups"); const [isCloudZeroModalOpen, setIsCloudZeroModalOpen] = useState(false); const [isGlobalExportModalOpen, setIsGlobalExportModalOpen] = useState(false); diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 6f716ccd66c..11e957ae6d4 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -57,7 +57,7 @@ interface MenuGroup { const Sidebar: React.FC = ({ setPage, defaultSelectedKey, collapsed = false }) => { const { userId, accessToken, userRole } = useAuthorized(); - const { data: organizations } = useOrganizations(accessToken); + const { data: organizations } = useOrganizations(); // Check if user is an org_admin const isOrgAdmin = useMemo(() => { diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx index 7830edf5867..ed429622ff8 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPServerSelector.tsx @@ -23,8 +23,8 @@ const MCPServerSelector: React.FC = ({ placeholder = "Select MCP servers", disabled = false, }) => { - const { data: mcpServers = [], isLoading: serversLoading } = useMCPServers(accessToken); - const { data: accessGroups = [], isLoading: groupsLoading } = useMCPAccessGroups(accessToken); + const { data: mcpServers = [], isLoading: serversLoading } = useMCPServers(); + const { data: accessGroups = [], isLoading: groupsLoading } = useMCPAccessGroups(); const loading = serversLoading || groupsLoading; diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx index fdd69d064d3..07b19cc5552 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.test.tsx @@ -71,7 +71,9 @@ describe("MCPToolPermissions", () => { }); // Verify API calls - expect(networking.fetchMCPServers).toHaveBeenCalledWith(mockAccessToken); + // Note: useMCPServers uses useAuthorized() internally, which returns "123" from global mock + expect(networking.fetchMCPServers).toHaveBeenCalledWith("123"); + // listMCPTools uses the accessToken prop directly expect(networking.listMCPTools).toHaveBeenCalledWith(mockAccessToken, mockServerId); }); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx index b6323397524..776d579fc13 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.test.tsx @@ -32,7 +32,7 @@ const createQueryClient = () => describe("MCPServers", () => { const defaultProps = { - accessToken: "test-token", + accessToken: "123", userRole: "Admin", userID: "admin-user-id", }; @@ -120,6 +120,7 @@ describe("MCPServers", () => { expect(getByText("test-server-2")).toBeInTheDocument(); // Verify the API was called - expect(networking.fetchMCPServers).toHaveBeenCalledWith("test-token"); + // Note: useMCPServers uses useAuthorized() internally, which returns "123" from global mock + expect(networking.fetchMCPServers).toHaveBeenCalledWith("123"); }); }); diff --git a/ui/litellm-dashboard/src/components/model_add/credentials.tsx b/ui/litellm-dashboard/src/components/model_add/credentials.tsx index 3887e340daa..af3c757955e 100644 --- a/ui/litellm-dashboard/src/components/model_add/credentials.tsx +++ b/ui/litellm-dashboard/src/components/model_add/credentials.tsx @@ -32,7 +32,7 @@ interface CredentialsPanelProps { const CredentialsPanel: React.FC = ({ uploadProps }) => { const { accessToken } = useAuthorized(); - const { data: credentialsResponse, refetch: refetchCredentials } = useCredentials(accessToken); + const { data: credentialsResponse, refetch: refetchCredentials } = useCredentials(); const credentialList = credentialsResponse?.credentials || []; const [isAddModalOpen, setIsAddModalOpen] = useState(false); diff --git a/ui/litellm-dashboard/tests/setupTests.ts b/ui/litellm-dashboard/tests/setupTests.ts index cce37d79057..e1ae45e8c58 100644 --- a/ui/litellm-dashboard/tests/setupTests.ts +++ b/ui/litellm-dashboard/tests/setupTests.ts @@ -33,6 +33,20 @@ vi.mock("@tremor/react", async (importOriginal) => { }; }); +// Global mock for useAuthorized hook to avoid repeating the same mock in every test file +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + token: "123", + accessToken: "123", + userId: "user-1", + userEmail: "user@example.com", + userRole: "Admin", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }), +})); + afterEach(() => { cleanup(); }); From 1ac620475606fb79e72632cfa02bbaac007abf75 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 10:19:57 -0800 Subject: [PATCH 062/388] fixing build --- ui/litellm-dashboard/src/components/OldTeams.tsx | 2 +- .../src/components/mcp_server_management/MCPToolPermissions.tsx | 2 +- ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx | 2 +- ui/litellm-dashboard/src/components/model_info_view.tsx | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index 77bbdb483da..10a38f0285c 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -174,7 +174,7 @@ const Teams: React.FC = ({ premiumUser = false, }) => { console.log(`organizations: ${JSON.stringify(organizations)}`); - const { data: organizationsData } = useOrganizations(accessToken); + const { data: organizationsData } = useOrganizations(); const [lastRefreshed, setLastRefreshed] = useState(""); const [currentOrg, setCurrentOrg] = useState(null); const [currentOrgForCreateTeam, setCurrentOrgForCreateTeam] = useState(null); diff --git a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx index ec7e2797814..4f884d3303b 100644 --- a/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx +++ b/ui/litellm-dashboard/src/components/mcp_server_management/MCPToolPermissions.tsx @@ -21,7 +21,7 @@ const MCPToolPermissions: React.FC = ({ onChange, disabled = false, }) => { - const { data: allServers = [] } = useMCPServers(accessToken); + const { data: allServers = [] } = useMCPServers(); const [serverTools, setServerTools] = useState>({}); const [loadingTools, setLoadingTools] = useState>({}); const [toolErrors, setToolErrors] = useState>({}); diff --git a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx index 83393c4a94b..d5b147f85c7 100644 --- a/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx +++ b/ui/litellm-dashboard/src/components/mcp_tools/mcp_servers.tsx @@ -19,7 +19,7 @@ const EDIT_OAUTH_UI_STATE_KEY = "litellm-mcp-oauth-edit-state"; const { Option } = Select; const MCPServers: React.FC = ({ accessToken, userRole, userID }) => { - const { data: mcpServers, isLoading: isLoadingServers, refetch, dataUpdatedAt } = useMCPServers(accessToken); + const { data: mcpServers, isLoading: isLoadingServers, refetch, dataUpdatedAt } = useMCPServers(); // Log allowed_tools from fetched servers React.useEffect(() => { diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index f66fd005ae1..01acc8d78ef 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -86,7 +86,7 @@ export default function ModelInfoView({ const isAdmin = userRole === "Admin"; const isAutoRouter = modelData?.litellm_params?.auto_router_config != null; - const { data: modelsInfoData } = useModelsInfo(accessToken, userID, userRole); + const { data: modelsInfoData } = useModelsInfo(); console.log("modelsInfoData, ", modelsInfoData); const usingExistingCredential = modelData?.litellm_params?.litellm_credential_name != null && From dc4982a8f37729078d33fe94e1a78da6309c2ce0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 10:48:53 -0800 Subject: [PATCH 063/388] Base commit --- litellm/proxy/management_endpoints/internal_user_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 1850ffa2560..9137b29ce27 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1467,7 +1467,7 @@ async def get_users( ), ): """ - Get a paginated list of users with filtering and sorting options. + Get a paginated list of users with filtering and sorting options Parameters: role: Optional[str] From 5a20edf0fa840a45e78977fa724001b5c45bb41b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 11:43:36 -0800 Subject: [PATCH 064/388] Allow org admins to view org info --- .../proxy/common_utils/http_parsing_utils.py | 13 +++- .../internal_user_endpoints.py | 2 +- .../common_utils/test_http_parsing_utils.py | 67 +++++++++++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 259755f5ef9..1d94b10f6a4 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -329,8 +329,8 @@ def populate_request_with_path_params( request_data: dict, request: Request ) -> dict: """ - Copy FastAPI path params into the request payload so downstream checks - (e.g. vector store RBAC) see them the same way as body params. + Copy FastAPI path params and query params into the request payload so downstream checks + (e.g. vector store RBAC, organization RBAC) see them the same way as body params. Since path_params may not be available during dependency injection, we parse the URL path directly for known patterns. @@ -340,8 +340,15 @@ def populate_request_with_path_params( request: The FastAPI Request object Returns: - dict: Updated request_data with path parameters added + dict: Updated request_data with path parameters and query parameters added """ + # Add query parameters to request_data (for GET requests, etc.) + query_params = _safe_get_request_query_params(request) + if query_params: + for key, value in query_params.items(): + # Don't overwrite existing values from request body + request_data.setdefault(key, value) + # Try to get path_params if available (sometimes populated by FastAPI) path_params = getattr(request, "path_params", None) if isinstance(path_params, dict) and path_params: diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 9137b29ce27..1850ffa2560 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1467,7 +1467,7 @@ async def get_users( ), ): """ - Get a paginated list of users with filtering and sorting options + Get a paginated list of users with filtering and sorting options. Parameters: role: Optional[str] diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 2361decc5af..324a58acfa9 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -24,6 +24,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( get_form_data, get_request_body, get_tags_from_request_body, + populate_request_with_path_params, ) @@ -630,3 +631,69 @@ def test_get_tags_from_request_body_with_null_metadata(): assert result == [] assert isinstance(result, list) + + +def test_populate_request_with_path_params_adds_query_params(): + """ + Test that populate_request_with_path_params correctly adds query parameters + like organization_id to the request data. + """ + # Create a mock request with query parameters + mock_request = MagicMock() + # Mock query_params as a dict-like object that can be converted to dict + mock_request.query_params = { + "organization_id": "org-123", + "user_id": "user-456" + } + mock_request.path_params = {} + # Mock url.path to avoid errors in _add_vector_store_id_from_path + mock_request.url.path = "/v1/chat/completions" + + # Initial request data without query params + request_data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "Hello"}] + } + + # Call the function + result = populate_request_with_path_params(request_data, mock_request) + + # Verify query params were added + assert result["organization_id"] == "org-123" + assert result["user_id"] == "user-456" + # Verify original data is preserved + assert result["model"] == "gpt-4" + assert result["messages"] == [{"role": "user", "content": "Hello"}] + + +def test_populate_request_with_path_params_does_not_overwrite_existing_values(): + """ + Test that populate_request_with_path_params does not overwrite existing values + in request_data when query params contain the same keys. + """ + # Create a mock request with query parameters + mock_request = MagicMock() + # Mock query_params as a dict-like object that can be converted to dict + mock_request.query_params = { + "organization_id": "org-query-param", + "model": "gpt-3.5-turbo" + } + mock_request.path_params = {} + # Mock url.path to avoid errors in _add_vector_store_id_from_path + mock_request.url.path = "/v1/chat/completions" + + # Initial request data with existing values + request_data = { + "model": "gpt-4", # This should NOT be overwritten + "organization_id": "org-existing", # This should NOT be overwritten + "messages": [{"role": "user", "content": "Hello"}] + } + + # Call the function + result = populate_request_with_path_params(request_data, mock_request) + + # Verify existing values were NOT overwritten + assert result["model"] == "gpt-4" # Should keep original, not "gpt-3.5-turbo" + assert result["organization_id"] == "org-existing" # Should keep original, not "org-query-param" + # Verify other data is preserved + assert result["messages"] == [{"role": "user", "content": "Hello"}] From c1dcc81fd500ef805df34c254e317c743eda77db Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 14:06:59 -0800 Subject: [PATCH 065/388] Model Page sort by all models --- .../ModelsAndEndpointsView.tsx | 1 - .../components/AllModelsTab.tsx | 25 ++++++++----------- .../src/components/model_dashboard/table.tsx | 21 +++++++++------- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx index 969c7bafa6b..bf001c62126 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/ModelsAndEndpointsView.tsx @@ -661,7 +661,6 @@ const ModelsAndEndpointsView: React.FC = ({ setSelectedModelId={setSelectedModelId} setSelectedTeamId={setSelectedTeamId} setEditModel={setEditModel} - modelData={modelData} /> {!shouldHideAddModelTab && ( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx index 87fa0b1e3b6..04c05ede5c2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx @@ -1,13 +1,14 @@ +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import useTeams from "@/app/(dashboard)/hooks/useTeams"; import { Team } from "@/components/key_team_helpers/key_list"; import { ModelDataTable } from "@/components/model_dashboard/table"; import { columns } from "@/components/molecules/models/columns"; import { getDisplayModelName } from "@/components/view_model/model_name_display"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { PaginationState, Table as TableInstance } from "@tanstack/react-table"; +import { PaginationState } from "@tanstack/react-table"; import { Grid, Select, SelectItem, TabPanel, Text } from "@tremor/react"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; +import { useModelsInfo } from "../../hooks/models/useModels"; type ModelViewMode = "all" | "current_team"; @@ -19,7 +20,6 @@ interface AllModelsTabProps { setSelectedModelId: (id: string) => void; setSelectedTeamId: (id: string) => void; setEditModel: (edit: boolean) => void; - modelData: any; } const AllModelsTab = ({ @@ -30,10 +30,10 @@ const AllModelsTab = ({ setSelectedModelId, setSelectedTeamId, setEditModel, - modelData, }: AllModelsTabProps) => { + const { data: modelData } = useModelsInfo(); const { userId, userRole, premiumUser } = useAuthorized(); - const { teams } = useTeams(); + const { data: teams } = useTeams(); const [modelNameSearch, setModelNameSearch] = useState(""); const [modelViewMode, setModelViewMode] = useState("current_team"); @@ -45,7 +45,6 @@ const AllModelsTab = ({ pageIndex: 0, pageSize: 50, }); - const tableRef = useRef>(null); const filteredData = useMemo(() => { if (!modelData || !modelData.data || modelData.data.length === 0) { @@ -88,12 +87,6 @@ const AllModelsTab = ({ }); }, [modelData, modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); - const paginatedData = useMemo(() => { - const startIndex = pagination.pageIndex * pagination.pageSize; - const endIndex = startIndex + pagination.pageSize; - return filteredData.slice(startIndex, endIndex); - }, [filteredData, pagination.pageIndex, pagination.pageSize]); - useEffect(() => { setPagination((prev: PaginationState) => ({ ...prev, pageIndex: 0 })); }, [modelNameSearch, selectedModelGroup, selectedModelAccessGroupFilter, currentTeam, modelViewMode]); @@ -370,9 +363,11 @@ const AllModelsTab = ({ expandedRows, setExpandedRows, )} - data={paginatedData} + data={filteredData} isLoading={false} - table={tableRef} + pagination={pagination} + onPaginationChange={setPagination} + enablePagination={true} /> diff --git a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx index 344ff2e94f2..79224edba43 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/table.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/table.tsx @@ -3,10 +3,13 @@ import { flexRender, getCoreRowModel, getSortedRowModel, + getPaginationRowModel, SortingState, useReactTable, ColumnResizeMode, VisibilityState, + PaginationState, + OnChangeFn, } from "@tanstack/react-table"; import React from "react"; import { Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; @@ -23,16 +26,20 @@ interface ModelDataTableProps { data: TData[]; columns: ColumnDef[]; isLoading?: boolean; - table: any; // Add table prop to access column visibility controls defaultSorting?: SortingState; + pagination?: PaginationState; + onPaginationChange?: OnChangeFn; + enablePagination?: boolean; } export function ModelDataTable({ data = [], columns, isLoading = false, - table, defaultSorting = [], + pagination, + onPaginationChange, + enablePagination = false, }: ModelDataTableProps) { const [sorting, setSorting] = React.useState(defaultSorting); const [columnResizeMode] = React.useState("onChange"); @@ -46,13 +53,16 @@ export function ModelDataTable({ sorting, columnSizing, columnVisibility, + ...(enablePagination && pagination ? { pagination } : {}), }, columnResizeMode, onSortingChange: setSorting, onColumnSizingChange: setColumnSizing, onColumnVisibilityChange: setColumnVisibility, + ...(enablePagination && onPaginationChange ? { onPaginationChange } : {}), getCoreRowModel: getCoreRowModel(), getSortedRowModel: getSortedRowModel(), + ...(enablePagination ? { getPaginationRowModel: getPaginationRowModel() } : {}), enableSorting: true, enableColumnResizing: true, defaultColumn: { @@ -61,13 +71,6 @@ export function ModelDataTable({ }, }); - // Expose table instance to parent - React.useEffect(() => { - if (table) { - table.current = tableInstance; - } - }, [tableInstance, table]); - const getHeaderText = (header: any): string => { if (typeof header === "string") { return header; From 5d1fe86cda3609071b29b4b3900cc58002184ff1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 14:33:27 -0800 Subject: [PATCH 066/388] Tests --- .../components/AllModelsTab.test.tsx | 103 ++++++++++++------ 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx index a4bb20128e0..dfa400e6ea9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx @@ -1,9 +1,27 @@ import * as useAuthorizedModule from "@/app/(dashboard)/hooks/useAuthorized"; -import * as useTeamsModule from "@/app/(dashboard)/hooks/useTeams"; import { render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import AllModelsTab from "./AllModelsTab"; +// Mock the useModelsInfo hook +const mockUseModelsInfo = vi.fn(() => ({ data: { data: [] } })) as any; + +vi.mock("../../hooks/models/useModels", () => ({ + useModelsInfo: () => mockUseModelsInfo(), +})); + +// Mock the useTeams hook (react-query implementation) +const mockUseTeams = vi.fn(() => ({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), +})) as any; + +vi.mock("../../hooks/teams/useTeams", () => ({ + useTeams: () => mockUseTeams(), +})); + describe("AllModelsTab", () => { const mockSetSelectedModelGroup = vi.fn(); const mockSetSelectedModelId = vi.fn(); @@ -18,9 +36,6 @@ describe("AllModelsTab", () => { setSelectedModelId: mockSetSelectedModelId, setSelectedTeamId: mockSetSelectedTeamId, setEditModel: mockSetEditModel, - modelData: { - data: [], - }, }; const mockUseAuthorized = { @@ -40,9 +55,13 @@ describe("AllModelsTab", () => { }); it("should render with empty data", () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseModelsInfo.mockReturnValueOnce({ data: { data: [] } }); + + mockUseTeams.mockReturnValueOnce({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); render(); @@ -66,9 +85,11 @@ describe("AllModelsTab", () => { }, ]; - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: mockTeams, - setTeams: vi.fn(), + mockUseTeams.mockReturnValueOnce({ + data: mockTeams, + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -92,7 +113,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Showing 0 results")).toBeInTheDocument(); @@ -116,9 +139,11 @@ describe("AllModelsTab", () => { }, ]; - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: mockTeams, - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: mockTeams, + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -142,7 +167,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Showing 0 results")).toBeInTheDocument(); @@ -150,9 +177,11 @@ describe("AllModelsTab", () => { }); it("should filter models by direct_access for personal team", async () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -178,7 +207,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Showing 1 - 1 of 1 results")).toBeInTheDocument(); @@ -186,9 +217,11 @@ describe("AllModelsTab", () => { }); it("should show config model status for models defined in configs", async () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { @@ -226,7 +259,9 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); + + render(); await waitFor(() => { expect(screen.getByText("Config Model")).toBeInTheDocument(); @@ -235,19 +270,21 @@ describe("AllModelsTab", () => { }); it("should show 'Defined in config' for models defined in configs", async () => { - vi.spyOn(useTeamsModule, "default").mockReturnValue({ - teams: [], - setTeams: vi.fn(), + mockUseTeams.mockReturnValue({ + data: [], + isLoading: false, + error: null, + refetch: vi.fn(), }); const modelData = { data: [ { - model_name: "gpt-4-config-model", - litellm_model_name: "gpt-4-config-model", + model_name: "gpt-4-config", + litellm_model_name: "gpt-4-config", provider: "openai", model_info: { - id: "model-config-defined", + id: "model-config-1", db_model: false, direct_access: true, access_via_team_ids: [], @@ -260,8 +297,12 @@ describe("AllModelsTab", () => { ], }; - render(); + mockUseModelsInfo.mockReturnValue({ data: modelData }); - expect(screen.getByText("Defined in config")).toBeInTheDocument(); + render(); + + await waitFor(() => { + expect(screen.getByText("Defined in config")).toBeInTheDocument(); + }); }); }); From 4ce135727f8b00e6fa1af077be15b2e2accc5983 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 14:51:00 -0800 Subject: [PATCH 067/388] Fixing build --- .../model_dashboard/HealthCheckComponent.tsx | 1 - .../src/components/model_hub_table.tsx | 9 +- .../src/components/public_model_hub.tsx | 37 +++----- .../components/templates/model_dashboard.tsx | 93 ++++++++++--------- 4 files changed, 66 insertions(+), 74 deletions(-) diff --git a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx index 994ea8adfc0..5d35b92684c 100644 --- a/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx +++ b/ui/litellm-dashboard/src/components/model_dashboard/HealthCheckComponent.tsx @@ -596,7 +596,6 @@ const HealthCheckComponent: React.FC = ({ }; })} isLoading={false} - table={healthTableRef} /> diff --git a/ui/litellm-dashboard/src/components/model_hub_table.tsx b/ui/litellm-dashboard/src/components/model_hub_table.tsx index 7d48bf68aed..f45e44ce905 100644 --- a/ui/litellm-dashboard/src/components/model_hub_table.tsx +++ b/ui/litellm-dashboard/src/components/model_hub_table.tsx @@ -1,10 +1,9 @@ import { CopyOutlined } from "@ant-design/icons"; -import { Table as TableInstance } from "@tanstack/react-table"; import { Badge, Button, Card, Tab, TabGroup, TabList, TabPanel, TabPanels, Text, Title } from "@tremor/react"; import { Modal } from "antd"; import { Copy } from "lucide-react"; import { useRouter } from "next/navigation"; -import React, { useCallback, useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import { isAdminRole } from "../utils/roles"; import { agentHubColumns, AgentHubData } from "./agent_hub_table_columns"; @@ -76,9 +75,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, const [isMcpModalVisible, setIsMcpModalVisible] = useState(false); const [isMakeMcpPublicModalVisible, setIsMakeMcpPublicModalVisible] = useState(false); const router = useRouter(); - const tableRef = useRef>(null); - const agentTableRef = useRef>(null); - const mcpTableRef = useRef>(null); useEffect(() => { const fetchData = async (accessToken: string) => { @@ -404,7 +400,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, columns={modelHubColumns(showModal, copyToClipboard, publicPage)} data={filteredData} isLoading={loading} - table={tableRef} defaultSorting={[{ id: "model_group", desc: false }]} /> @@ -431,7 +426,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, columns={agentHubColumns(showAgentModal, copyToClipboard, publicPage)} data={agentHubData || []} isLoading={agentLoading} - table={agentTableRef} defaultSorting={[{ id: "name", desc: false }]} /> @@ -458,7 +452,6 @@ const ModelHubTable: React.FC = ({ accessToken, publicPage, columns={mcpHubColumns(showMcpModal, copyToClipboard, publicPage)} data={mcpHubData || []} isLoading={mcpLoading} - table={mcpTableRef} defaultSorting={[{ id: "server_name", desc: false }]} /> diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 3493f0bf93f..4678dbe3f94 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -1,25 +1,24 @@ -import React, { useEffect, useState, useRef, useMemo } from "react"; -import { - modelHubPublicModelsCall, - getPublicModelHubInfo, - agentHubPublicModelsCall, - mcpHubPublicServersCall, - getUiConfig, -} from "./networking"; -import { ModelDataTable } from "./model_dashboard/table"; -import { ColumnDef } from "@tanstack/react-table"; -import { Card, Text, Title, Button } from "@tremor/react"; -import { Tag, Tooltip, Modal, Select, Tabs } from "antd"; +import { ThemeProvider } from "@/contexts/ThemeContext"; import { ExternalLinkIcon, SearchIcon } from "@heroicons/react/outline"; +import { ColumnDef } from "@tanstack/react-table"; +import { Button, Card, Text, Title } from "@tremor/react"; +import { Modal, Select, Tabs, Tag, Tooltip } from "antd"; import { Copy, Info } from "lucide-react"; -import { Table as TableInstance } from "@tanstack/react-table"; +import React, { useEffect, useMemo, useState } from "react"; +import { ModelDataTable } from "./model_dashboard/table"; +import NotificationsManager from "./molecules/notifications_manager"; +import Navbar from "./navbar"; +import { + agentHubPublicModelsCall, + getPublicModelHubInfo, + getUiConfig, + mcpHubPublicServersCall, + modelHubPublicModelsCall, +} from "./networking"; import { generateCodeSnippet } from "./playground/chat_ui/CodeSnippets"; import { getEndpointType } from "./playground/chat_ui/mode_endpoint_mapping"; import { MessageType } from "./playground/chat_ui/types"; import { getProviderLogoAndName } from "./provider_info_helpers"; -import Navbar from "./navbar"; -import { ThemeProvider } from "@/contexts/ThemeContext"; -import NotificationsManager from "./molecules/notifications_manager"; const { TabPane } = Tabs; @@ -118,9 +117,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded const [selectedMcpServer, setSelectedMcpServer] = useState(null); const [proxySettings, setProxySettings] = useState({}); const [activeTab, setActiveTab] = useState("models"); - const tableRef = useRef>(null); - const agentTableRef = useRef>(null); - const mcpTableRef = useRef>(null); useEffect(() => { const initializeAndFetch = async () => { @@ -1121,7 +1117,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded columns={publicModelHubColumns()} data={filteredData} isLoading={loading} - table={tableRef} defaultSorting={[{ id: "model_group", desc: false }]} /> @@ -1184,7 +1179,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded columns={publicAgentHubColumns()} data={filteredAgentData} isLoading={agentLoading} - table={agentTableRef} defaultSorting={[{ id: "name", desc: false }]} /> @@ -1248,7 +1242,6 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded columns={publicMCPHubColumns()} data={filteredMcpData} isLoading={mcpLoading} - table={mcpTableRef} defaultSorting={[{ id: "server_name", desc: false }]} /> diff --git a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx index 6d43e0af057..9dbe04bffb1 100644 --- a/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/templates/model_dashboard.tsx @@ -1,65 +1,74 @@ -import React, { useState, useEffect, useRef, useMemo } from "react"; import { Card, - Title, + Col, + Grid, Subtitle, Table, - TableHead, - TableRow, - TableHeaderCell, - TableCell, TableBody, + TableCell, + TableHead, + TableHeaderCell, + TableRow, Text, - Grid, - Col, + Title, } from "@tremor/react"; +import React, { useEffect, useMemo, useRef, useState } from "react"; import { CredentialItem, credentialListCall, CredentialsResponse } from "../networking"; import { handleAddModelSubmit } from "../add_model/handle_add_model_submit"; import CredentialsPanel from "@/components/model_add/credentials"; -import { getDisplayModelName } from "../view_model/model_name_display"; -import { TabPanel, TabPanels, TabGroup, TabList, Tab, Icon } from "@tremor/react"; -import { Select, SelectItem, DateRangePickerValue } from "@tremor/react"; -import UsageDatePicker from "../shared/usage_date_picker"; +import { InfoCircleOutlined } from "@ant-design/icons"; +import { FilterIcon, RefreshIcon } from "@heroicons/react/outline"; +import { + AreaChart, + BarChart, + Button, + DateRangePickerValue, + Icon, + Select, + SelectItem, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, +} from "@tremor/react"; +import type { UploadProps } from "antd"; +import { Form, InputNumber, Popover, Typography } from "antd"; +import AddModelTab from "../add_model/add_model_tab"; +import { Team } from "../key_team_helpers/key_list"; +import ModelInfoView from "../model_info_view"; +import TimeToFirstToken from "../model_metrics/time_to_first_token"; import { - modelInfoCall, - modelCostMap, - healthCheckCall, - modelMetricsCall, - streamingModelMetricsCall, - modelExceptionsCall, - modelMetricsSlowResponsesCall, - getCallbacksCall, - setCallbacksCall, - modelSettingsCall, adminGlobalActivityExceptions, adminGlobalActivityExceptionsPerDeployment, allEndUsersCall, + getCallbacksCall, + healthCheckCall, + modelCostMap, + modelExceptionsCall, + modelInfoCall, + modelMetricsCall, + modelMetricsSlowResponsesCall, + modelSettingsCall, + setCallbacksCall, + streamingModelMetricsCall, } from "../networking"; -import { BarChart, AreaChart } from "@tremor/react"; -import { Popover, Form, InputNumber } from "antd"; -import { Button } from "@tremor/react"; -import { Typography } from "antd"; -import { RefreshIcon, FilterIcon } from "@heroicons/react/outline"; -import { InfoCircleOutlined } from "@ant-design/icons"; -import type { UploadProps } from "antd"; -import TimeToFirstToken from "../model_metrics/time_to_first_token"; -import { Team } from "../key_team_helpers/key_list"; +import { getPlaceholder, getProviderModels, provider_map, Providers } from "../provider_info_helpers"; +import UsageDatePicker from "../shared/usage_date_picker"; import TeamInfoView from "../team/team_info"; -import { Providers, provider_map, getPlaceholder, getProviderModels } from "../provider_info_helpers"; -import ModelInfoView from "../model_info_view"; -import AddModelTab from "../add_model/add_model_tab"; +import { getDisplayModelName } from "../view_model/model_name_display"; -import { ModelDataTable } from "../model_dashboard/table"; -import { columns } from "../molecules/models/columns"; -import PriceDataReload from "../price_data_reload"; -import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; -import PassThroughSettings from "../pass_through_settings"; -import ModelGroupAliasSettings from "../model_group_alias_settings"; import { all_admin_roles } from "@/utils/roles"; -import { Table as TableInstance, PaginationState } from "@tanstack/react-table"; +import { PaginationState } from "@tanstack/react-table"; +import HealthCheckComponent from "../model_dashboard/HealthCheckComponent"; +import { ModelDataTable } from "../model_dashboard/table"; +import ModelGroupAliasSettings from "../model_group_alias_settings"; +import { columns } from "../molecules/models/columns"; import NotificationsManager from "../molecules/notifications_manager"; +import PassThroughSettings from "../pass_through_settings"; +import PriceDataReload from "../price_data_reload"; interface ModelDashboardProps { accessToken: string | null; @@ -196,7 +205,6 @@ const OldModelDashboard: React.FC = ({ const [isDropdownOpen, setIsDropdownOpen] = useState(false); const [expandedRows, setExpandedRows] = useState>(new Set()); const dropdownRef = useRef(null); - const tableRef = useRef>(null); // Pagination state const [pagination, setPagination] = useState({ @@ -1325,7 +1333,6 @@ const OldModelDashboard: React.FC = ({ )} data={paginatedData} isLoading={false} - table={tableRef} /> From 85827aa217962d415ff72ffee41e650dfedb3a27 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Wed, 24 Dec 2025 17:12:36 -0800 Subject: [PATCH 068/388] Resize columns working --- .../src/components/all_keys_table.tsx | 75 ++++++++++++++++++- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/all_keys_table.tsx b/ui/litellm-dashboard/src/components/all_keys_table.tsx index a915fe06179..210ce09fa34 100644 --- a/ui/litellm-dashboard/src/components/all_keys_table.tsx +++ b/ui/litellm-dashboard/src/components/all_keys_table.tsx @@ -1,6 +1,6 @@ "use client"; import React, { useEffect, useState } from "react"; -import { ColumnDef } from "@tanstack/react-table"; +import { ColumnDef, ColumnResizeMode, ColumnResizeDirection } from "@tanstack/react-table"; import { Select, SelectItem } from "@tremor/react"; import { Button } from "@tremor/react"; import KeyInfoView from "./templates/key_info_view"; @@ -125,6 +125,8 @@ export function AllKeysTable({ }: AllKeysTableProps) { const [selectedKeyId, setSelectedKeyId] = useState(null); const [userList, setUserList] = useState([]); + const [columnResizeMode, setColumnResizeMode] = React.useState("onChange"); + const [columnResizeDirection, setColumnResizeDirection] = React.useState("ltr"); const [sorting, setSorting] = React.useState(() => { if (currentSort) { return [ @@ -184,6 +186,7 @@ export function AllKeysTable({ { id: "expander", header: () => null, + size: 40, cell: ({ row }) => row.getCanExpand() ? ( + )} + + + + {isSSOConfigured ? ( + renderSSOSettings() + ) : ( + setIsAddModalVisible(true)} /> + )} + + + setIsDeleteModalVisible(false)} + onSuccess={() => refetch()} + accessToken={accessToken} + /> + + setIsAddModalVisible(false)} + onSuccess={() => { + setIsAddModalVisible(false); + refetch(); + }} + accessToken={accessToken} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.tsx new file mode 100644 index 00000000000..fc315493a54 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.tsx @@ -0,0 +1,30 @@ +import { Empty, Typography, Button } from "antd"; + +const { Title, Paragraph } = Typography; + +interface SSOSettingsEmptyPlaceholderProps { + onAdd: () => void; +} + +export default function SSOSettingsEmptyPlaceholder({ onAdd }: SSOSettingsEmptyPlaceholderProps) { + return ( +
+ + No SSO Configuration Found + + Configure Single Sign-On (SSO) to enable seamless authentication for your team members using your identity + provider. + +
+ } + > + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/admins.tsx b/ui/litellm-dashboard/src/components/admins.tsx index 4ddd5cd5d1f..6af5a226da6 100644 --- a/ui/litellm-dashboard/src/components/admins.tsx +++ b/ui/litellm-dashboard/src/components/admins.tsx @@ -55,6 +55,7 @@ import { getSSOSettings, } from "./networking"; import UISettings from "./Settings/AdminSettings/UISettings/UISettings"; +import SSOSettings from "./Settings/AdminSettings/SSOSettings/SSOSettings"; const AdminPanel: React.FC = ({ searchParams, @@ -496,11 +497,15 @@ const AdminPanel: React.FC = ({ Go to 'Internal Users' page to add other admins. + SSO Settings Security Settings SCIM UI Settings + + + ✨ Security Settings From 546fba98498d8019d8ba911e2a4ee62dd35df1a7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 2 Jan 2026 18:28:10 -0800 Subject: [PATCH 205/388] tests --- .../Modals/AddSSOSettingsModal.test.tsx | 26 +++++++++++++ .../Modals/DeleteSSOSettingsModal.test.tsx | 20 ++++++++++ .../SSOSettings/SSOSettings.test.tsx | 37 +++++++++++++++++++ .../SSOSettingsEmptyPlaceholder.test.tsx | 14 +++++++ 4 files changed, 97 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.test.tsx diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx new file mode 100644 index 00000000000..13363a11643 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx @@ -0,0 +1,26 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import AddSSOSettingsModal from "./AddSSOSettingsModal"; + +// Mock networking functions +vi.mock("@/components/networking", () => ({ + updateSSOSettings: vi.fn(), +})); + +// Mock error utils +vi.mock("@/components/shared/errorUtils", () => ({ + parseErrorMessage: vi.fn((error) => error?.message || "Unknown error"), +})); + +describe("AddSSOSettingsModal", () => { + it("should render", () => { + const onCancel = vi.fn(); + const onSuccess = vi.fn(); + + render(); + + expect(screen.getByText("SSO Provider")).toBeInTheDocument(); + expect(screen.getByText("Cancel")).toBeInTheDocument(); + expect(screen.getAllByText("Add SSO")).toHaveLength(2); // Title and button + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx new file mode 100644 index 00000000000..ef6ec6c7055 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/DeleteSSOSettingsModal.test.tsx @@ -0,0 +1,20 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import DeleteSSOSettingsModal from "./DeleteSSOSettingsModal"; + +describe("DeleteSSOSettingsModal", () => { + it("should render", () => { + const onCancel = vi.fn(); + const onSuccess = vi.fn(); + + render( + , + ); + + expect(screen.getByText("Confirm Clear SSO Settings")).toBeInTheDocument(); + expect( + screen.getByText("Are you sure you want to clear all SSO settings? This action cannot be undone."), + ).toBeInTheDocument(); + expect(screen.getByText("Users will no longer be able to login using SSO after this change.")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx new file mode 100644 index 00000000000..5e7908a872b --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.test.tsx @@ -0,0 +1,37 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import SSOSettings from "./SSOSettings"; + +// Mock the useSSOSettings hook +vi.mock("@/app/(dashboard)/hooks/sso/useSSOSettings", () => ({ + useSSOSettings: () => ({ + data: null, + refetch: vi.fn(), + }), +})); + +const createQueryClient = () => + new QueryClient({ + defaultOptions: { + queries: { + retry: false, + gcTime: 0, + }, + }, + }); + +describe("SSOSettings", () => { + it("should render", () => { + const queryClient = createQueryClient(); + + render( + + + , + ); + + expect(screen.getByText("SSO Configuration")).toBeInTheDocument(); + expect(screen.getByText("Manage Single Sign-On authentication settings")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.test.tsx new file mode 100644 index 00000000000..6676ba1c2c9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsEmptyPlaceholder.test.tsx @@ -0,0 +1,14 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; + +describe("SSOSettingsEmptyPlaceholder", () => { + it("should render", () => { + const onAdd = vi.fn(); + + render(); + + expect(screen.getByText("No SSO Configuration Found")).toBeInTheDocument(); + expect(screen.getByText("Configure SSO")).toBeInTheDocument(); + }); +}); From f4c712506dc9b139b2e0b0e46865d3fd3a109121 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 2 Jan 2026 18:49:01 -0800 Subject: [PATCH 206/388] Unit tests to increase test coverage --- .../hooks/uiSettings/useUISettings.test.ts | 185 +++++++ .../playground/chat_ui/EndpointUtils.test.tsx | 219 ++++++++ .../prompt_editor_view/ToolsCard.test.tsx | 82 +++ .../VersionHistorySidePanel.test.tsx | 473 ++++++++++++++++++ .../prompts/prompt_editor_view/utils.test.ts | 444 ++++++++++++++++ 5 files changed, 1403 insertions(+) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useUISettings.test.ts create mode 100644 ui/litellm-dashboard/src/components/playground/chat_ui/EndpointUtils.test.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ToolsCard.test.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/VersionHistorySidePanel.test.tsx create mode 100644 ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useUISettings.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useUISettings.test.ts new file mode 100644 index 00000000000..785f003d2f8 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/uiSettings/useUISettings.test.ts @@ -0,0 +1,185 @@ +import { getUiSettings } from "@/components/networking"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { renderHook, waitFor } from "@testing-library/react"; +import React, { ReactNode } from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useUISettings } from "./useUISettings"; + +// Mock the networking function +vi.mock("@/components/networking", () => ({ + getUiSettings: vi.fn(), +})); + +// Mock useAuthorized hook - we can override this in individual tests +const mockUseAuthorized = vi.fn(); +vi.mock("../useAuthorized", () => ({ + default: () => mockUseAuthorized(), +})); + +// Mock data +const mockUISettings: Record = { + theme: "dark", + language: "en", + notifications: true, + dashboard_layout: "compact", + api_keys_visible: false, +}; + +describe("useUISettings", () => { + let queryClient: QueryClient; + + beforeEach(() => { + queryClient = new QueryClient({ + defaultOptions: { + queries: { + retry: false, + }, + }, + }); + + // Reset all mocks + vi.clearAllMocks(); + + // Set default mock for useAuthorized (enabled state) + mockUseAuthorized.mockReturnValue({ + accessToken: "test-access-token", + userRole: "Admin", + userId: "test-user-id", + token: "test-token", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + }); + + const wrapper = ({ children }: { children: ReactNode }) => + React.createElement(QueryClientProvider, { client: queryClient }, children); + + it("should return UI settings data when query is successful", async () => { + // Mock successful API call + (getUiSettings as any).mockResolvedValue(mockUISettings); + + const { result } = renderHook(() => useUISettings(), { wrapper }); + + // Initially loading + expect(result.current.isLoading).toBe(true); + expect(result.current.data).toBeUndefined(); + + // Wait for success + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data).toEqual(mockUISettings); + expect(result.current.error).toBeNull(); + expect(getUiSettings).toHaveBeenCalledWith("test-access-token"); + expect(getUiSettings).toHaveBeenCalledTimes(1); + }); + + it("should handle error when getUiSettings fails", async () => { + const errorMessage = "Failed to fetch UI settings"; + const testError = new Error(errorMessage); + + // Mock failed API call + (getUiSettings as any).mockRejectedValue(testError); + + const { result } = renderHook(() => useUISettings(), { wrapper }); + + // Initially loading + expect(result.current.isLoading).toBe(true); + + // Wait for error + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isError).toBe(true); + }); + + expect(result.current.error).toEqual(testError); + expect(result.current.data).toBeUndefined(); + expect(getUiSettings).toHaveBeenCalledWith("test-access-token"); + expect(getUiSettings).toHaveBeenCalledTimes(1); + }); + + it("should not execute query when accessToken is missing", async () => { + // Mock missing accessToken + mockUseAuthorized.mockReturnValue({ + accessToken: null, + userRole: "Admin", + userId: "test-user-id", + token: null, + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useUISettings(), { wrapper }); + + // Query should not execute + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + + // API should not be called + expect(getUiSettings).not.toHaveBeenCalled(); + }); + + it("should not execute query when accessToken is empty string", async () => { + // Mock empty accessToken + mockUseAuthorized.mockReturnValue({ + accessToken: "", + userRole: "Admin", + userId: "test-user-id", + token: "", + userEmail: "test@example.com", + premiumUser: false, + disabledPersonalKeyCreation: null, + showSSOBanner: false, + }); + + const { result } = renderHook(() => useUISettings(), { wrapper }); + + // Query should not execute + expect(result.current.isLoading).toBe(false); + expect(result.current.data).toBeUndefined(); + expect(result.current.isFetched).toBe(false); + + // API should not be called + expect(getUiSettings).not.toHaveBeenCalled(); + }); + + it("should return empty object when API returns empty settings", async () => { + // Mock API returning empty object + (getUiSettings as any).mockResolvedValue({}); + + const { result } = renderHook(() => useUISettings(), { wrapper }); + + // Wait for success + await waitFor(() => { + expect(result.current.isLoading).toBe(false); + expect(result.current.isSuccess).toBe(true); + }); + + expect(result.current.data).toEqual({}); + expect(getUiSettings).toHaveBeenCalledWith("test-access-token"); + }); + + it("should handle network timeout error", async () => { + const timeoutError = new Error("Network timeout"); + + // Mock network timeout + (getUiSettings as any).mockRejectedValue(timeoutError); + + const { result } = renderHook(() => useUISettings(), { wrapper }); + + // Wait for error + await waitFor(() => { + expect(result.current.isError).toBe(true); + }); + + expect(result.current.error).toEqual(timeoutError); + expect(result.current.data).toBeUndefined(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/EndpointUtils.test.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/EndpointUtils.test.tsx new file mode 100644 index 00000000000..6eb481381ac --- /dev/null +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/EndpointUtils.test.tsx @@ -0,0 +1,219 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { ModelGroup } from "../llm_calls/fetch_models"; +import { determineEndpointType } from "./EndpointUtils"; +import { EndpointType } from "./mode_endpoint_mapping"; + +// Mock the getEndpointType function +vi.mock("./mode_endpoint_mapping", () => ({ + EndpointType: { + IMAGE: "image", + VIDEO: "video", + CHAT: "chat", + RESPONSES: "responses", + IMAGE_EDITS: "image_edits", + ANTHROPIC_MESSAGES: "anthropic_messages", + EMBEDDINGS: "embeddings", + SPEECH: "speech", + TRANSCRIPTION: "transcription", + A2A_AGENTS: "a2a_agents", + }, + getEndpointType: vi.fn(), + ModelMode: { + AUDIO_SPEECH: "audio_speech", + AUDIO_TRANSCRIPTION: "audio_transcription", + IMAGE_GENERATION: "image_generation", + VIDEO_GENERATION: "video_generation", + CHAT: "chat", + RESPONSES: "responses", + IMAGE_EDITS: "image_edits", + ANTHROPIC_MESSAGES: "anthropic_messages", + EMBEDDING: "embedding", + }, +})); + +// Import the mocked function +import { getEndpointType } from "./mode_endpoint_mapping"; + +describe("determineEndpointType", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should return the correct endpoint type when model is found and has a valid mode", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "gpt-3.5-turbo", + mode: "chat", + }, + { + model_group: "dall-e-3", + mode: "image_generation", + }, + ]; + + // Mock getEndpointType to return IMAGE for image_generation mode + vi.mocked(getEndpointType).mockReturnValue(EndpointType.IMAGE); + + const result = determineEndpointType("dall-e-3", mockModelInfo); + + expect(getEndpointType).toHaveBeenCalledWith("image_generation"); + expect(result).toBe(EndpointType.IMAGE); + }); + + it("should return CHAT endpoint type when model is found but has no mode", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "gpt-3.5-turbo", + // No mode property + }, + ]; + + const result = determineEndpointType("gpt-3.5-turbo", mockModelInfo); + + expect(getEndpointType).not.toHaveBeenCalled(); + expect(result).toBe(EndpointType.CHAT); + }); + + it("should return CHAT endpoint type when model is not found in modelInfo", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "gpt-3.5-turbo", + mode: "chat", + }, + ]; + + const result = determineEndpointType("non-existent-model", mockModelInfo); + + expect(getEndpointType).not.toHaveBeenCalled(); + expect(result).toBe(EndpointType.CHAT); + }); + + it("should return CHAT endpoint type when modelInfo array is empty", () => { + const mockModelInfo: ModelGroup[] = []; + + const result = determineEndpointType("any-model", mockModelInfo); + + expect(getEndpointType).not.toHaveBeenCalled(); + expect(result).toBe(EndpointType.CHAT); + }); + + it("should handle different mode types correctly", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "tts-model", + mode: "audio_speech", + }, + { + model_group: "whisper-model", + mode: "audio_transcription", + }, + { + model_group: "embedding-model", + mode: "embedding", + }, + { + model_group: "video-model", + mode: "video_generation", + }, + ]; + + // Test speech mode + vi.mocked(getEndpointType).mockReturnValueOnce(EndpointType.SPEECH); + const speechResult = determineEndpointType("tts-model", mockModelInfo); + expect(getEndpointType).toHaveBeenCalledWith("audio_speech"); + expect(speechResult).toBe(EndpointType.SPEECH); + + // Reset mock for next test + vi.clearAllMocks(); + + // Test transcription mode + vi.mocked(getEndpointType).mockReturnValueOnce(EndpointType.TRANSCRIPTION); + const transcriptionResult = determineEndpointType("whisper-model", mockModelInfo); + expect(getEndpointType).toHaveBeenCalledWith("audio_transcription"); + expect(transcriptionResult).toBe(EndpointType.TRANSCRIPTION); + + // Reset mock for next test + vi.clearAllMocks(); + + // Test embedding mode + vi.mocked(getEndpointType).mockReturnValueOnce(EndpointType.EMBEDDINGS); + const embeddingResult = determineEndpointType("embedding-model", mockModelInfo); + expect(getEndpointType).toHaveBeenCalledWith("embedding"); + expect(embeddingResult).toBe(EndpointType.EMBEDDINGS); + + // Reset mock for next test + vi.clearAllMocks(); + + // Test video mode + vi.mocked(getEndpointType).mockReturnValueOnce(EndpointType.VIDEO); + const videoResult = determineEndpointType("video-model", mockModelInfo); + expect(getEndpointType).toHaveBeenCalledWith("video_generation"); + expect(videoResult).toBe(EndpointType.VIDEO); + }); + + it("should prioritize the first matching model when there are duplicates", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "gpt-3.5-turbo", + mode: "chat", + }, + { + model_group: "gpt-3.5-turbo", + mode: "image_generation", // Different mode for same model name + }, + ]; + + vi.mocked(getEndpointType).mockReturnValue(EndpointType.CHAT); + + const result = determineEndpointType("gpt-3.5-turbo", mockModelInfo); + + expect(getEndpointType).toHaveBeenCalledWith("chat"); + expect(result).toBe(EndpointType.CHAT); + }); + + it("should handle models with undefined mode property explicitly set", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "test-model", + mode: undefined, + }, + ]; + + const result = determineEndpointType("test-model", mockModelInfo); + + expect(getEndpointType).not.toHaveBeenCalled(); + expect(result).toBe(EndpointType.CHAT); + }); + + it("should handle models with empty string mode", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "test-model", + mode: "", + }, + ]; + + const result = determineEndpointType("test-model", mockModelInfo); + + // Empty string is falsy, so getEndpointType should not be called + expect(getEndpointType).not.toHaveBeenCalled(); + expect(result).toBe(EndpointType.CHAT); + }); + + it("should handle case-sensitive model group matching", () => { + const mockModelInfo: ModelGroup[] = [ + { + model_group: "GPT-3.5-TURBO", + mode: "chat", + }, + ]; + + vi.mocked(getEndpointType).mockReturnValue(EndpointType.CHAT); + + // Test with different case - should not match + const result = determineEndpointType("gpt-3.5-turbo", mockModelInfo); + + expect(getEndpointType).not.toHaveBeenCalled(); + expect(result).toBe(EndpointType.CHAT); + }); +}); diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ToolsCard.test.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ToolsCard.test.tsx new file mode 100644 index 00000000000..742b8aa37fc --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/ToolsCard.test.tsx @@ -0,0 +1,82 @@ +import { act, fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import ToolsCard from "./ToolsCard"; +import { Tool } from "./types"; + +describe("ToolsCard", () => { + const mockTools: Tool[] = [ + { + name: "Calculator", + description: "Performs mathematical calculations", + json: '{"type": "function", "function": {"name": "calculate"}}', + }, + { + name: "Weather API", + description: "Gets current weather information", + json: '{"type": "function", "function": {"name": "get_weather"}}', + }, + ]; + + const defaultProps = { + tools: [] as Tool[], + onAddTool: vi.fn(), + onEditTool: vi.fn(), + onRemoveTool: vi.fn(), + }; + + it("should render the component", () => { + render(); + expect(screen.getByText("Tools")).toBeInTheDocument(); + }); + + it("should display no tools message when tools array is empty", () => { + render(); + expect(screen.getByText("No tools added")).toBeInTheDocument(); + }); + + it("should render tools when provided", () => { + render(); + + expect(screen.getByText("Calculator")).toBeInTheDocument(); + expect(screen.getByText("Performs mathematical calculations")).toBeInTheDocument(); + expect(screen.getByText("Weather API")).toBeInTheDocument(); + expect(screen.getByText("Gets current weather information")).toBeInTheDocument(); + }); + + it("should call onAddTool when Add button is clicked", () => { + const mockOnAddTool = vi.fn(); + render(); + + act(() => { + fireEvent.click(screen.getByRole("button", { name: /add/i })); + }); + + expect(mockOnAddTool).toHaveBeenCalledTimes(1); + }); + + it("should call onEditTool with correct index when Edit button is clicked", () => { + const mockOnEditTool = vi.fn(); + render(); + + const editButtons = screen.getAllByText("Edit"); + act(() => { + fireEvent.click(editButtons[0]); + }); + + expect(mockOnEditTool).toHaveBeenCalledWith(0); + expect(mockOnEditTool).toHaveBeenCalledTimes(1); + }); + + it("should call onRemoveTool with correct index when remove button is clicked", () => { + const mockOnRemoveTool = vi.fn(); + render(); + + const removeButtons = screen.getAllByRole("button", { name: "" }); + act(() => { + fireEvent.click(removeButtons[0]); + }); + + expect(mockOnRemoveTool).toHaveBeenCalledWith(0); + expect(mockOnRemoveTool).toHaveBeenCalledTimes(1); + }); +}); diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/VersionHistorySidePanel.test.tsx b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/VersionHistorySidePanel.test.tsx new file mode 100644 index 00000000000..b0346e03a2d --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/VersionHistorySidePanel.test.tsx @@ -0,0 +1,473 @@ +import { describe, it, expect, vi, beforeEach, afterEach, type Mock } from "vitest"; +import { render, screen, fireEvent, act, waitFor } from "@testing-library/react"; +import VersionHistorySidePanel from "./VersionHistorySidePanel"; +import { getPromptVersions } from "../../networking"; +import type { PromptSpec } from "../../networking"; + +// Mock the networking function +vi.mock("../../networking", () => ({ + getPromptVersions: vi.fn(), +})); + +const mockGetPromptVersions = getPromptVersions as Mock; + +// Mock Ant Design components that might need special handling +vi.mock("antd", async () => { + const actual = await vi.importActual("antd"); + return { + ...actual, + Drawer: ({ children, title, onClose, open, width, placement, mask, maskClosable }: any) => ( +
+
{title}
+ +
{children}
+
+ ), + List: ({ children, dataSource, renderItem }: any) => ( +
{dataSource?.map((item: any, index: number) => renderItem(item, index))}
+ ), + Skeleton: ({ active }: any) => ( +
+ Loading... +
+ ), + Tag: ({ children, color, className }: any) => ( + + {children} + + ), + Typography: { + Text: ({ children, type, className }: any) => ( + + {children} + + ), + }, + }; +}); + +describe("VersionHistorySidePanel", () => { + // Mock data + const mockPromptVersions: PromptSpec[] = [ + { + prompt_id: "test-prompt.v2", + litellm_params: { prompt_id: "test-prompt.v2" }, + prompt_info: { prompt_type: "db" }, + version: 2, + created_at: "2024-01-15T10:30:00Z", + }, + { + prompt_id: "test-prompt.v1", + litellm_params: { prompt_id: "test-prompt.v1" }, + prompt_info: { prompt_type: "db" }, + version: 1, + created_at: "2024-01-10T09:00:00Z", + }, + { + prompt_id: "test-prompt.v3", + litellm_params: { prompt_id: "test-prompt.v3" }, + prompt_info: { prompt_type: "config" }, + version: 3, + created_at: "2024-01-20T14:15:00Z", + }, + ]; + + const mockPromptVersionsWithoutExplicitVersion = [ + { + prompt_id: "test-prompt.v2", + litellm_params: { prompt_id: "test-prompt.v2" }, + prompt_info: { prompt_type: "db" }, + created_at: "2024-01-15T10:30:00Z", + }, + { + prompt_id: "test-prompt.v1", + litellm_params: { prompt_id: "test-prompt.v1" }, + prompt_info: { prompt_type: "db" }, + created_at: "2024-01-10T09:00:00Z", + }, + ]; + + const defaultProps = { + isOpen: true, + onClose: vi.fn(), + accessToken: "test-token", + promptId: "test-prompt.v2", + activeVersionId: "test-prompt.v2", + onSelectVersion: vi.fn(), + }; + + beforeEach(() => { + vi.clearAllMocks(); + // Mock successful response by default + mockGetPromptVersions.mockResolvedValue({ + prompts: mockPromptVersions, + }); + }); + + afterEach(() => { + vi.clearAllTimers(); + }); + + describe("Component Rendering", () => { + it("should render the component with drawer", async () => { + await act(async () => { + render(); + }); + expect(screen.getByTestId("drawer")).toBeInTheDocument(); + expect(screen.getByText("Version History")).toBeInTheDocument(); + }); + + it("should not render when isOpen is false", async () => { + await act(async () => { + render(); + }); + // The drawer should still be rendered but with open=false + const drawer = screen.getByTestId("drawer"); + expect(drawer).toHaveAttribute("data-open", "false"); + }); + + it("should show loading skeleton initially", async () => { + // Mock a delayed response to show loading state + mockGetPromptVersions.mockImplementationOnce( + () => new Promise((resolve) => setTimeout(() => resolve({ prompts: mockPromptVersions }), 100)), + ); + + render(); + expect(screen.getByTestId("skeleton")).toBeInTheDocument(); + + // Wait for loading to complete + await waitFor(() => { + expect(screen.queryByTestId("skeleton")).not.toBeInTheDocument(); + }); + }); + + it("should show empty state when no versions are available", async () => { + mockGetPromptVersions.mockResolvedValueOnce({ prompts: [] }); + + render(); + + await waitFor(() => { + expect(screen.getByText("No version history available.")).toBeInTheDocument(); + }); + }); + + it("should render version list when data is loaded", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("v2")).toBeInTheDocument(); + expect(screen.getByText("v1")).toBeInTheDocument(); + expect(screen.getByText("v3")).toBeInTheDocument(); + }); + + // Check that Latest tag is shown for the first item + const latestTags = screen.getAllByText("Latest"); + expect(latestTags.length).toBeGreaterThan(0); + + // Check Active tag is shown for the active version + expect(screen.getByText("Active")).toBeInTheDocument(); + }); + }); + + describe("Version Selection and Highlighting", () => { + it("should highlight the active version correctly", async () => { + render(); + + await waitFor(() => { + const versionItems = screen.getAllByTestId("tag"); + // Should have Active tag for the selected version + expect(screen.getByText("Active")).toBeInTheDocument(); + }); + }); + + it("should highlight the latest version when no activeVersionId is provided", async () => { + render(); + + await waitFor(() => { + const latestTags = screen.getAllByText("Latest"); + expect(latestTags.length).toBeGreaterThan(0); + }); + }); + + it("should call onSelectVersion when a version is clicked", async () => { + const mockOnSelectVersion = vi.fn(); + render(); + + await waitFor(() => { + expect(screen.getByText("v1")).toBeInTheDocument(); + }); + + const versionItem = screen.getByText("v1").closest("div"); + expect(versionItem).toBeInTheDocument(); + + act(() => { + fireEvent.click(versionItem!); + }); + + expect(mockOnSelectVersion).toHaveBeenCalledWith(mockPromptVersions[1]); + }); + }); + + describe("Version Number Extraction", () => { + it("should extract version from explicit version field", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("v2")).toBeInTheDocument(); + expect(screen.getByText("v3")).toBeInTheDocument(); + }); + }); + + it("should extract version from prompt_id with .v suffix", async () => { + mockGetPromptVersions.mockResolvedValueOnce({ + prompts: mockPromptVersionsWithoutExplicitVersion, + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("v2")).toBeInTheDocument(); + expect(screen.getByText("v1")).toBeInTheDocument(); + }); + }); + + it("should extract version from prompt_id with _v suffix", async () => { + const versionsWithUnderscore = [ + { + prompt_id: "test-prompt_v2", + litellm_params: { prompt_id: "test-prompt_v2" }, + prompt_info: { prompt_type: "db" }, + created_at: "2024-01-15T10:30:00Z", + }, + ]; + + mockGetPromptVersions.mockResolvedValueOnce({ + prompts: versionsWithUnderscore, + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("v2")).toBeInTheDocument(); + }); + }); + + it("should default to v1 when no version info is available", async () => { + const versionWithoutVersionInfo = [ + { + prompt_id: "test-prompt", + litellm_params: { prompt_id: "test-prompt" }, + prompt_info: { prompt_type: "db" }, + created_at: "2024-01-15T10:30:00Z", + }, + ]; + + mockGetPromptVersions.mockResolvedValueOnce({ + prompts: versionWithoutVersionInfo, + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("v1")).toBeInTheDocument(); + }); + }); + }); + + describe("Date Formatting", () => { + it("should format dates correctly", async () => { + render(); + + await waitFor(() => { + // Check that dates are displayed (format: YYYY-MM-DD HH:MM:SS) + const dateElements = screen.getAllByTestId("text"); + const dateText = dateElements.find((el) => el.textContent?.match(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/)); + expect(dateText).toBeTruthy(); + }); + }); + + it("should show dash for missing dates", async () => { + const versionsWithoutDates = [ + { + prompt_id: "test-prompt.v1", + litellm_params: { prompt_id: "test-prompt.v1" }, + prompt_info: { prompt_type: "db" }, + version: 1, + }, + ]; + + mockGetPromptVersions.mockResolvedValueOnce({ + prompts: versionsWithoutDates, + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("-")).toBeInTheDocument(); + }); + }); + }); + + describe("Prompt Type Display", () => { + it("should show 'Saved to Database' for db prompts", async () => { + render(); + + await waitFor(() => { + const dbTexts = screen.getAllByText("Saved to Database"); + expect(dbTexts.length).toBeGreaterThan(0); + }); + }); + + it("should show 'Config Prompt' for config prompts", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Config Prompt")).toBeInTheDocument(); + }); + }); + }); + + describe("Network Calls and Data Fetching", () => { + it("should call getPromptVersions with correct parameters", async () => { + render(); + + await waitFor(() => { + expect(getPromptVersions).toHaveBeenCalledWith("test-token", "test-prompt"); + }); + }); + + it("should strip .v suffix from promptId when fetching versions", async () => { + render(); + + await waitFor(() => { + expect(getPromptVersions).toHaveBeenCalledWith("test-token", "test-prompt"); + }); + }); + + it("should not fetch versions when isOpen is false", () => { + render(); + + expect(getPromptVersions).not.toHaveBeenCalled(); + }); + + it("should not fetch versions when accessToken is null", () => { + render(); + + expect(getPromptVersions).not.toHaveBeenCalled(); + }); + + it("should not fetch versions when promptId is not provided", () => { + render(); + + expect(getPromptVersions).not.toHaveBeenCalled(); + }); + + it("should refetch versions when props change", async () => { + const { rerender } = render(); + + await waitFor(() => { + expect(getPromptVersions).toHaveBeenCalledTimes(1); + }); + + rerender(); + + await waitFor(() => { + expect(getPromptVersions).toHaveBeenCalledTimes(2); + expect(getPromptVersions).toHaveBeenCalledWith("test-token", "different-prompt"); + }); + }); + }); + + describe("Error Handling", () => { + it("should handle network errors gracefully", async () => { + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mockGetPromptVersions.mockRejectedValueOnce(new Error("Network error")); + + render(); + + await waitFor(() => { + expect(consoleSpy).toHaveBeenCalledWith("Error fetching prompt versions:", expect.any(Error)); + }); + + // Should show empty state when there's an error + expect(screen.getByText("No version history available.")).toBeInTheDocument(); + + consoleSpy.mockRestore(); + }); + }); + + describe("User Interactions", () => { + it("should call onClose when close button is clicked", () => { + const mockOnClose = vi.fn(); + render(); + + const drawer = screen.getByTestId("drawer"); + act(() => { + fireEvent.click(drawer); // Simulate close action + }); + + // Note: This test assumes the drawer handles close events. + // In a real scenario, you'd test the actual close trigger. + }); + + it("should prevent interaction with main content when drawer is open", () => { + render(); + + const drawer = screen.getByTestId("drawer"); + // The mask and maskClosable props are passed as boolean false to disable them + expect(drawer).toHaveAttribute("data-mask", "false"); + expect(drawer).toHaveAttribute("data-maskclosable", "false"); + }); + }); + + describe("Edge Cases", () => { + it("should handle activeVersionId with .v suffix correctly", async () => { + render(); + + await waitFor(() => { + expect(screen.getByText("Active")).toBeInTheDocument(); + }); + }); + + it("should handle activeVersionId with _v suffix correctly", async () => { + const versionsWithUnderscore = [ + { + prompt_id: "test-prompt_v2", + litellm_params: { prompt_id: "test-prompt_v2" }, + prompt_info: { prompt_type: "db" }, + version: 2, + created_at: "2024-01-15T10:30:00Z", + }, + ]; + + mockGetPromptVersions.mockResolvedValueOnce({ + prompts: versionsWithUnderscore, + }); + + render(); + + await waitFor(() => { + expect(screen.getByText("Active")).toBeInTheDocument(); + }); + }); + + it("should sort versions correctly with version field", async () => { + // The component doesn't explicitly sort, but we can verify the order from the API response + render(); + + await waitFor(() => { + const versionElements = screen.getAllByTestId("tag"); + // Verify versions are displayed as they come from the API + expect(screen.getByText("v2")).toBeInTheDocument(); + }); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.test.ts b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.test.ts new file mode 100644 index 00000000000..59fcaccb639 --- /dev/null +++ b/ui/litellm-dashboard/src/components/prompts/prompt_editor_view/utils.test.ts @@ -0,0 +1,444 @@ +import { describe, expect, it } from "vitest"; +import { PromptType } from "./types"; +import { + convertToDotPrompt, + extractVariables, + getVersionNumber, + parseExistingPrompt, + stripVersionFromPromptId, +} from "./utils"; + +describe("extractVariables", () => { + it("should extract variables from messages", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "", + messages: [ + { role: "user", content: "Hello {{name}}, how are you?" }, + { role: "assistant", content: "I am fine {{name}}" }, + ], + }; + + const result = extractVariables(prompt); + expect(result).toEqual(["name"]); + }); + + it("should extract variables from developer message", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "You are {{role}} assistant", + messages: [{ role: "user", content: "Hello" }], + }; + + const result = extractVariables(prompt); + expect(result).toEqual(["role"]); + }); + + it("should extract variables from both messages and developer message", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "You are {{role}} assistant", + messages: [ + { role: "user", content: "Hello {{name}}" }, + { role: "assistant", content: "Hi {{name}}, I am {{role}}" }, + ], + }; + + const result = extractVariables(prompt); + expect(result.sort()).toEqual(["name", "role"].sort()); + }); + + it("should return empty array when no variables present", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "You are an assistant", + messages: [{ role: "user", content: "Hello world" }], + }; + + const result = extractVariables(prompt); + expect(result).toEqual([]); + }); + + it("should handle duplicate variables", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "", + messages: [ + { role: "user", content: "Hello {{name}}" }, + { role: "assistant", content: "Hi {{name}} again" }, + ], + }; + + const result = extractVariables(prompt); + expect(result).toEqual(["name"]); + }); +}); + +describe("convertToDotPrompt", () => { + it("should convert basic prompt to dot prompt format", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "", + messages: [{ role: "user", content: "Hello world" }], + }; + + const result = convertToDotPrompt(prompt); + expect(result).toContain("---"); + expect(result).toContain("model: gpt-4"); + expect(result).toContain("input:"); + expect(result).toContain("schema:"); + expect(result).toContain("output:"); + expect(result).toContain("format: text"); + expect(result).toContain("User: Hello world"); + }); + + it("should include config parameters when set", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: { + temperature: 0.7, + max_tokens: 100, + top_p: 0.9, + }, + tools: [], + developerMessage: "", + messages: [{ role: "user", content: "Hello" }], + }; + + const result = convertToDotPrompt(prompt); + expect(result).toContain("temperature: 0.7"); + expect(result).toContain("max_tokens: 100"); + expect(result).toContain("top_p: 0.9"); + }); + + it("should include input schema with variables", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "", + messages: [{ role: "user", content: "Hello {{name}}" }], + }; + + const result = convertToDotPrompt(prompt); + expect(result).toContain("input:"); + expect(result).toContain("schema:"); + expect(result).toContain("name: string"); + }); + + it("should include developer message when present", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "You are a helpful assistant", + messages: [{ role: "user", content: "Hello" }], + }; + + const result = convertToDotPrompt(prompt); + expect(result).toContain("Developer: You are a helpful assistant"); + }); + + it("should include tools when present", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [ + { + name: "get_weather", + description: "Get weather information", + json: '{"type": "function", "function": {"name": "get_weather"}}', + }, + ], + developerMessage: "", + messages: [{ role: "user", content: "Hello" }], + }; + + const result = convertToDotPrompt(prompt); + expect(result).toContain("tools:"); + expect(result).toContain('{"type":"function","function":{"name":"get_weather"}}'); + }); + + it("should handle multiple messages with different roles", () => { + const prompt: PromptType = { + name: "test", + model: "gpt-4", + config: {}, + tools: [], + developerMessage: "", + messages: [ + { role: "user", content: "Hello" }, + { role: "assistant", content: "Hi there" }, + { role: "user", content: "How are you?" }, + ], + }; + + const result = convertToDotPrompt(prompt); + expect(result).toContain("User: Hello"); + expect(result).toContain("Assistant: Hi there"); + expect(result).toContain("User: How are you?"); + }); +}); + +describe("parseExistingPrompt", () => { + it("should parse basic dotprompt content", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: `--- +model: gpt-4 +input: + schema: +output: + format: text +--- + +User: Hello world`, + }, + prompt_id: "test-prompt", + }, + }; + + const result = parseExistingPrompt(apiResponse); + expect(result.name).toBe("test-prompt"); + expect(result.model).toBe("gpt-4"); + expect(result.messages).toEqual([{ role: "user", content: "Hello world" }]); + }); + + it("should parse with config parameters", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: `--- +model: gpt-4 +temperature: 0.7 +max_tokens: 100 +top_p: 0.9 +input: + schema: +output: + format: text +--- + +User: Hello`, + }, + prompt_id: "test-prompt", + }, + }; + + const result = parseExistingPrompt(apiResponse); + expect(result.config.temperature).toBe(0.7); + expect(result.config.max_tokens).toBe(100); + expect(result.config.top_p).toBe(0.9); + }); + + it("should parse with developer message", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: `--- +model: gpt-4 +input: + schema: +output: + format: text +--- + +Developer: You are a helpful assistant + +User: Hello`, + }, + prompt_id: "test-prompt", + }, + }; + + const result = parseExistingPrompt(apiResponse); + expect(result.developerMessage).toBe("You are a helpful assistant"); + }); + + it("should parse multiple messages", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: `--- +model: gpt-4 +input: + schema: +output: + format: text +--- + +User: Hello +How are you? + +Assistant: I am fine +Thank you for asking + +User: Great!`, + }, + prompt_id: "test-prompt", + }, + }; + + const result = parseExistingPrompt(apiResponse); + expect(result.messages).toEqual([ + { role: "user", content: "Hello\nHow are you?" }, + { role: "assistant", content: "I am fine\nThank you for asking" }, + { role: "user", content: "Great!" }, + ]); + }); + + it("should handle prompt with version suffix", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: `--- +model: gpt-4 +input: + schema: +output: + format: text +--- + +User: Hello`, + }, + prompt_id: "test-prompt.v2", + }, + }; + + const result = parseExistingPrompt(apiResponse); + expect(result.name).toBe("test-prompt"); + }); + + it("should throw error when no dotprompt_content", () => { + const apiResponse = { + prompt_spec: { + litellm_params: {}, + }, + }; + + expect(() => parseExistingPrompt(apiResponse)).toThrow("No dotprompt_content found in API response"); + }); + + it("should throw error for invalid dotprompt format", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: "invalid format", + }, + }, + }; + + expect(() => parseExistingPrompt(apiResponse)).toThrow("Invalid dotprompt format"); + }); + + it("should provide default values when parsing fails", () => { + const apiResponse = { + prompt_spec: { + litellm_params: { + dotprompt_content: `--- +model: gpt-4 +input: + schema: +output: + format: text +--- + +`, + }, + prompt_id: "test-prompt", + }, + }; + + const result = parseExistingPrompt(apiResponse); + expect(result.messages).toEqual([ + { role: "user", content: "Enter task specifics. Use {{template_variables}} for dynamic inputs" }, + ]); + }); +}); + +describe("getVersionNumber", () => { + it("should return '1' for undefined promptId", () => { + const result = getVersionNumber(undefined); + expect(result).toBe("1"); + }); + + it("should return '1' for promptId without version", () => { + const result = getVersionNumber("test-prompt"); + expect(result).toBe("1"); + }); + + it("should extract version with dot separator", () => { + const result = getVersionNumber("test-prompt.v2"); + expect(result).toBe("2"); + }); + + it("should extract version with underscore separator", () => { + const result = getVersionNumber("test-prompt_v3"); + expect(result).toBe("3"); + }); + + it("should extract version with hyphen separator", () => { + const result = getVersionNumber("test-prompt-v4"); + expect(result).toBe("4"); + }); + + it("should extract multi-digit version", () => { + const result = getVersionNumber("test-prompt.v123"); + expect(result).toBe("123"); + }); +}); + +describe("stripVersionFromPromptId", () => { + it("should return empty string for undefined promptId", () => { + const result = stripVersionFromPromptId(undefined); + expect(result).toBe(""); + }); + + it("should return promptId unchanged when no version present", () => { + const result = stripVersionFromPromptId("test-prompt"); + expect(result).toBe("test-prompt"); + }); + + it("should strip version with dot separator", () => { + const result = stripVersionFromPromptId("test-prompt.v2"); + expect(result).toBe("test-prompt"); + }); + + it("should strip version with underscore separator", () => { + const result = stripVersionFromPromptId("test-prompt_v3"); + expect(result).toBe("test-prompt"); + }); + + it("should strip version with hyphen separator", () => { + const result = stripVersionFromPromptId("test-prompt-v4"); + expect(result).toBe("test-prompt"); + }); + + it("should strip multi-digit version", () => { + const result = stripVersionFromPromptId("test-prompt.v123"); + expect(result).toBe("test-prompt"); + }); +}); From e6da33dc4ac9f3c39c2b58cb113d2e4df62ec8f0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 2 Jan 2026 19:22:56 -0800 Subject: [PATCH 207/388] Remove modal in useful links --- .../AIHub/UsefulLinksManagement.test.tsx | 158 +++++++++++++++++- .../AIHub/UsefulLinksManagement.tsx | 90 +++++----- 2 files changed, 189 insertions(+), 59 deletions(-) diff --git a/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.test.tsx b/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.test.tsx index 7b0651ad2d6..0a859ca95f8 100644 --- a/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.test.tsx @@ -1,10 +1,9 @@ +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { getProxyBaseUrl, getPublicModelHubInfo, updateUsefulLinksCall } from "@/components/networking"; import { render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; -import { Modal } from "antd"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import UsefulLinksManagement from "./UsefulLinksManagement"; -import NotificationsManager from "@/components/molecules/notifications_manager"; -import { getPublicModelHubInfo, updateUsefulLinksCall, getProxyBaseUrl } from "@/components/networking"; vi.mock("@/components/networking", () => ({ getPublicModelHubInfo: vi.fn(), @@ -25,8 +24,6 @@ const mockedUpdateUsefulLinksCall = vi.mocked(updateUsefulLinksCall); const mockedGetProxyBaseUrl = vi.mocked(getProxyBaseUrl); const mockedNotifications = vi.mocked(NotificationsManager); -let modalSuccessSpy: any; - describe("UsefulLinksManagement", () => { beforeEach(() => { mockedGetPublicModelHubInfo.mockResolvedValue({ @@ -37,11 +34,9 @@ describe("UsefulLinksManagement", () => { }); mockedUpdateUsefulLinksCall.mockResolvedValue({}); mockedGetProxyBaseUrl.mockReturnValue("https://proxy.example.com"); - modalSuccessSpy = vi.spyOn(Modal, "success").mockImplementation(() => ({ destroy: vi.fn() }) as any); }); afterEach(() => { - modalSuccessSpy.mockRestore(); vi.clearAllMocks(); }); @@ -108,4 +103,153 @@ describe("UsefulLinksManagement", () => { expect(mockedNotifications.success).toHaveBeenCalledWith("Link order saved successfully"); }); + + it("should display the Model Hub link", async () => { + render(); + + expect(await screen.findByRole("link", { name: /public model hub/i })).toBeInTheDocument(); + }); + + it("should edit a link when edit button is clicked", async () => { + const user = userEvent.setup(); + mockedGetPublicModelHubInfo.mockResolvedValue({ + docs_title: "Docs", + custom_docs_description: null, + litellm_version: "1.0.0", + useful_links: { + "Test Link": "https://test.example.com", + }, + }); + + render(); + + await waitFor(() => expect(screen.getByText("Test Link")).toBeInTheDocument()); + + // Click edit button + const editButton = screen.getByTestId("edit-link-0-Test Link"); + await user.click(editButton); + + // Should show input fields in edit mode + expect(screen.getByDisplayValue("Test Link")).toBeInTheDocument(); + expect(screen.getByDisplayValue("https://test.example.com")).toBeInTheDocument(); + }); + + it("should update a link when save is clicked in edit mode", async () => { + const user = userEvent.setup(); + mockedGetPublicModelHubInfo.mockResolvedValue({ + docs_title: "Docs", + custom_docs_description: null, + litellm_version: "1.0.0", + useful_links: { + "Test Link": "https://test.example.com", + }, + }); + + render(); + + await waitFor(() => expect(screen.getByText("Test Link")).toBeInTheDocument()); + + // Click edit button + const editButton = screen.getByTestId("edit-link-0-Test Link"); + await user.click(editButton); + + // Update the display name + const displayNameInput = screen.getByDisplayValue("Test Link"); + await user.clear(displayNameInput); + await user.type(displayNameInput, "Updated Link"); + + // Click save + await user.click(screen.getByRole("button", { name: /save/i })); + + await waitFor(() => + expect(mockedUpdateUsefulLinksCall).toHaveBeenCalledWith("token", { + "Updated Link": { url: "https://test.example.com", index: 0 }, + }), + ); + + expect(mockedNotifications.success).toHaveBeenCalledWith("Link updated successfully"); + }); + + it("should cancel editing when cancel button is clicked", async () => { + const user = userEvent.setup(); + mockedGetPublicModelHubInfo.mockResolvedValue({ + docs_title: "Docs", + custom_docs_description: null, + litellm_version: "1.0.0", + useful_links: { + "Test Link": "https://test.example.com", + }, + }); + + render(); + + await waitFor(() => expect(screen.getByText("Test Link")).toBeInTheDocument()); + + // Click edit button + const editButton = screen.getByTestId("edit-link-0-Test Link"); + await user.click(editButton); + + // Update the display name + const displayNameInput = screen.getByDisplayValue("Test Link"); + await user.clear(displayNameInput); + await user.type(displayNameInput, "Updated Link"); + + // Click cancel + await user.click(screen.getByRole("button", { name: /cancel/i })); + + // Should go back to normal view + expect(screen.getByText("Test Link")).toBeInTheDocument(); + expect(screen.queryByDisplayValue("Updated Link")).not.toBeInTheDocument(); + }); + + it("should not move down the last item in rearrange mode", async () => { + const user = userEvent.setup(); + mockedGetPublicModelHubInfo.mockResolvedValue({ + docs_title: "Docs", + custom_docs_description: null, + litellm_version: "1.0.0", + useful_links: { + "First Link": "https://first.example.com", + "Second Link": "https://second.example.com", + }, + }); + + render(); + + await waitFor(() => expect(screen.getByText("First Link")).toBeInTheDocument()); + + // Enter rearrange mode + await user.click(screen.getByRole("button", { name: /rearrange order/i })); + + // Try to move down the last item (should not do anything) + const secondLinkMoveDownButton = screen.getByTestId("move-down-1-Second Link"); + await user.click(secondLinkMoveDownButton); + + // Links should remain in same order + const linksAfter = screen.getAllByText(/First Link|Second Link/); + expect(linksAfter[0]).toHaveTextContent("First Link"); + expect(linksAfter[1]).toHaveTextContent("Second Link"); + }); + + it("should expand and collapse the component", async () => { + const user = userEvent.setup(); + render(); + + await waitFor(() => expect(screen.getByText("Link Management")).toBeInTheDocument()); + + // Initially expanded + expect(screen.getByText("Manage Existing Links")).toBeInTheDocument(); + + // Click to collapse + await user.click(screen.getByText("Link Management")); + + // Should be collapsed + expect(screen.queryByText("Manage Existing Links")).not.toBeInTheDocument(); + + // Click to expand again + await user.click(screen.getByText("Link Management")); + + // Should be expanded + expect(screen.getByText("Manage Existing Links")).toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.tsx b/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.tsx index 220113ded13..c73eaf52384 100644 --- a/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/UsefulLinksManagement.tsx @@ -1,11 +1,11 @@ -import React, { useState, useEffect } from "react"; -import { Modal } from "antd"; -import { PlusCircleIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; -import { isAdminRole } from "@/utils/roles"; -import { getPublicModelHubInfo, updateUsefulLinksCall, getProxyBaseUrl } from "../networking"; -import { Card, Title, Text, Table, TableHead, TableHeaderCell, TableBody, TableRow, TableCell } from "@tremor/react"; -import NotificationsManager from "@/components/molecules/notifications_manager"; import TableIconActionButton from "@/components/common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { isAdminRole } from "@/utils/roles"; +import { ChevronDownIcon, ChevronRightIcon, ExternalLinkIcon, PlusCircleIcon } from "@heroicons/react/outline"; +import { Card, Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Text, Title } from "@tremor/react"; +import Link from "next/link"; +import React, { useEffect, useState } from "react"; +import { getProxyBaseUrl, getPublicModelHubInfo, updateUsefulLinksCall } from "../networking"; interface UsefulLinksManagementProps { accessToken: string | null; @@ -102,32 +102,6 @@ const UsefulLinksManagement: React.FC = ({ accessTok }); await updateUsefulLinksCall(accessToken, linksObject); - // show success modal with public model hub link - Modal.success({ - title: "Links Saved Successfully", - content: ( -
-

- Your useful links have been saved and are now visible on the public model hub. -

-
-

View your updated model hub:

- - Open Public Model Hub → - -
-
- ), - width: 500, - okText: "Close", - maskClosable: true, - keyboard: true, - }); return true; } catch (error) { @@ -319,29 +293,41 @@ const UsefulLinksManagement: React.FC = ({ accessTok
Manage Existing Links - {!isRearranging ? ( - - ) : ( -
+ Public Model Hub + + + {!isRearranging ? ( - -
- )} + ) : ( +
+ + +
+ )} +
From 0aae5153b6b59f9bbe6f479e70e4b97eaafa8365 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Sat, 3 Jan 2026 16:06:07 +0530 Subject: [PATCH 208/388] docs: Clarify Bedrock AgentCore documentation (#18603) Co-authored-by: Cursor Agent --- docs/my-website/docs/providers/bedrock_agentcore.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/my-website/docs/providers/bedrock_agentcore.md b/docs/my-website/docs/providers/bedrock_agentcore.md index 43df7f82519..e3e352f7ab6 100644 --- a/docs/my-website/docs/providers/bedrock_agentcore.md +++ b/docs/my-website/docs/providers/bedrock_agentcore.md @@ -11,6 +11,12 @@ Call Bedrock AgentCore in the OpenAI Request/Response format. | Provider Route on LiteLLM | `bedrock/agentcore/{AGENT_RUNTIME_ARN}` | | Provider Doc | [AWS Bedrock AgentCore ↗](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) | +:::info + +This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details. + +::: + ## Quick Start ### Model Format to LiteLLM From 87fe62229f4b8b5dddefa7c22521eb5662928ca1 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Sat, 3 Jan 2026 21:51:19 +0530 Subject: [PATCH 209/388] feat: Add adopters page and data structure (#18605) Co-authored-by: Cursor Agent --- docs/my-website/src/data/adopters/README.md | 88 +++++++++++++++++++ .../src/data/adopters/adopters.json | 8 ++ docs/my-website/src/data/adopters/index.js | 23 +++++ .../img/adopters/placeholder-company.svg | 8 ++ 4 files changed, 127 insertions(+) create mode 100644 docs/my-website/src/data/adopters/README.md create mode 100644 docs/my-website/src/data/adopters/adopters.json create mode 100644 docs/my-website/src/data/adopters/index.js create mode 100644 docs/my-website/static/img/adopters/placeholder-company.svg diff --git a/docs/my-website/src/data/adopters/README.md b/docs/my-website/src/data/adopters/README.md new file mode 100644 index 00000000000..61a5215f802 --- /dev/null +++ b/docs/my-website/src/data/adopters/README.md @@ -0,0 +1,88 @@ +# LiteLLM Adopters + +This directory contains data for organizations that use LiteLLM in production. + +## Adding Your Organization + +We've made it super easy to add your organization! Just follow the steps below. + +### Quick Add (Recommended) + +**[Edit adopters.json on GitHub →](https://github.com/BerriAI/litellm/edit/main/docs/my-website/src/data/adopters/adopters.json)** + +This will open the GitHub editor in your browser where you can: + +1. Add your organization's entry to the JSON array +2. Commit your changes +3. GitHub will automatically create a pull request for you! + +No need to clone the repository or set up a development environment. + +### JSON Format + +Add your organization to the array in `adopters.json`: + +```json +{ + "name": "Your Organization Name", + "logoUrl": "https://yoursite.com/logo.svg", + "url": "https://yourcompany.com", + "description": "Brief description of how you use LiteLLM (shown on hover)" +} +``` + +### Fields + +- **`name`** (required): Your organization's display name +- **`logoUrl`** (required): URL to your logo - can be either: + - External URL: `https://yoursite.com/logo.svg` (easiest!) + - Local path: `/img/adopters/your-logo.svg` (requires uploading logo file) +- **`url`** (optional): Your organization's website (makes the logo clickable) +- **`description`** (optional): Brief description shown when users hover over your logo + +### Logo Options + +#### Option 1: External URL (Easiest) + +Simply provide a direct link to your logo hosted anywhere: + +```json +"logoUrl": "https://yourcompany.com/assets/logo.svg" +``` + +#### Option 2: Local Logo (Better Performance) + +If you prefer to host the logo locally: + +1. Add your logo to `docs/my-website/static/img/adopters/your-company.svg` +2. Reference it as: `"logoUrl": "/img/adopters/your-company.svg"` + +**Logo Specifications:** + +- **Format**: SVG preferred (PNG also acceptable) +- **Dimensions**: 240x160px or similar 3:2 ratio recommended +- **Background**: Transparent or white background works best + +### Example + +```json +{ + "name": "Acme Corporation", + "logoUrl": "https://acme.com/logo.svg", + "url": "https://acme.com", + "description": "Using LiteLLM to route requests across 50+ LLM providers" +} +``` + +### Display Order + +Adopters are displayed alphabetically by organization name, so your position will be determined automatically. + +### Need Help? + +If you have questions about adding your organization: + +- Ask in [GitHub Discussions](https://github.com/BerriAI/litellm/discussions) +- Join our [Discord community](https://discord.com/invite/wuPM9dRgDw) + +Thank you for supporting LiteLLM! 🚅 diff --git a/docs/my-website/src/data/adopters/adopters.json b/docs/my-website/src/data/adopters/adopters.json new file mode 100644 index 00000000000..52319c149e2 --- /dev/null +++ b/docs/my-website/src/data/adopters/adopters.json @@ -0,0 +1,8 @@ +[ + { + "name": "Your Logo Here", + "logoUrl": "/img/adopters/placeholder-company.svg", + "description": "Add your organization to show support for LiteLLM", + "url": "https://github.com/BerriAI/litellm/edit/main/docs/my-website/src/data/adopters/adopters.json" + } +] diff --git a/docs/my-website/src/data/adopters/index.js b/docs/my-website/src/data/adopters/index.js new file mode 100644 index 00000000000..b1a242dcc33 --- /dev/null +++ b/docs/my-website/src/data/adopters/index.js @@ -0,0 +1,23 @@ +import adoptersData from './adopters.json'; + +/** + * @typedef {Object} Adopter + * @property {string} name - The organization's display name + * @property {string} logoUrl - URL to the organization's logo + * @property {string} [url] - The organization's website URL + * @property {string} [description] - Brief description shown on hover + */ + +/** + * List of organizations using LiteLLM + * @type {Adopter[]} + */ +export const adopters = adoptersData; + +/** + * Adopters sorted alphabetically by name + * @type {Adopter[]} + */ +export const sortedAdopters = [...adopters].sort((a, b) => + a.name.localeCompare(b.name) +); diff --git a/docs/my-website/static/img/adopters/placeholder-company.svg b/docs/my-website/static/img/adopters/placeholder-company.svg new file mode 100644 index 00000000000..937dffc6eaf --- /dev/null +++ b/docs/my-website/static/img/adopters/placeholder-company.svg @@ -0,0 +1,8 @@ + + + + + + Add Your Logo + Click to contribute + From bdd05475bca872f944a7cf704d20cd3fdb72e0cc Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Sat, 3 Jan 2026 15:39:00 -0300 Subject: [PATCH 210/388] fix: correct cost calculation when reasoning_tokens present without text_tokens (#18607) Fixes #18599 When OpenAI models (gpt-5-nano, o1-*, o3-*) and other providers return reasoning_tokens in completion_tokens_details but don't provide text_tokens, LiteLLM was incorrectly calculating costs using only reasoning_tokens, ignoring the remaining completion tokens. Changes: - Modified generic_cost_per_token() in llm_cost_calc/utils.py to calculate text_tokens as: completion_tokens - reasoning_tokens - audio_tokens - image_tokens when text_tokens is not explicitly provided - Added comprehensive test case test_reasoning_tokens_without_text_tokens_gpt5_nano() to verify all completion_tokens are billed correctly Example: - completion_tokens: 977 - reasoning_tokens: 768 - Before: only 768 tokens billed (21% less) - After: all 977 tokens billed correctly Affected models: - OpenAI: gpt-5-nano, o1-*, o3-* - Perplexity: sonar-reasoning* - Any model returning reasoning_tokens without text_tokens --- .../litellm_core_utils/llm_cost_calc/utils.py | 20 ++++++-- .../llm_cost_calc/test_llm_cost_calc_utils.py | 51 +++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e36d6d68367..cbc0763382c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -604,12 +604,22 @@ def generic_cost_per_token( reasoning_tokens = completion_tokens_details["reasoning_tokens"] image_tokens = completion_tokens_details["image_tokens"] - # Only assume all tokens are text if there's NO breakdown at all - # If image_tokens, audio_tokens, or reasoning_tokens exist, respect text_tokens=0 + # Handle text_tokens calculation: + # 1. If text_tokens is explicitly provided and > 0, use it + # 2. If there's a breakdown (reasoning/audio/image tokens), calculate text_tokens as the remainder + # 3. If no breakdown at all, assume all completion_tokens are text_tokens has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 - if text_tokens == 0 and not has_token_breakdown: - text_tokens = usage.completion_tokens - is_text_tokens_total = True + if text_tokens == 0: + if has_token_breakdown: + # Calculate text tokens as remainder when we have a breakdown + # This handles cases like OpenAI's reasoning models where text_tokens isn't provided + text_tokens = max( + 0, usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens + ) + else: + # No breakdown at all, all tokens are text tokens + text_tokens = usage.completion_tokens + is_text_tokens_total = True ## TEXT COST completion_cost = float(text_tokens) * completion_base_cost diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 65e3dbec8bd..5ba78d9eed1 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -809,3 +809,54 @@ def test_bedrock_anthropic_prompt_caching(): assert completion_cost >= 0 assert round(prompt_cost, 3) == 0.111 assert round(completion_cost, 5) == 0.00820 + + +def test_reasoning_tokens_without_text_tokens_gpt5_nano(): + """ + Test fix for GitHub issue #18599: + https://github.com/BerriAI/litellm/issues/18599 + + When OpenAI models (gpt-5-nano, o1, o3) return reasoning_tokens but don't provide + text_tokens, LiteLLM should calculate text_tokens as: + text_tokens = completion_tokens - reasoning_tokens - audio_tokens - image_tokens + + This ensures ALL completion tokens are billed, not just reasoning tokens. + """ + model = "gpt-5-nano" + custom_llm_provider = "openai" + + # Simulate OpenAI gpt-5-nano response where text_tokens is NOT provided + # completion_tokens: 977 total + # reasoning_tokens: 768 + # text_tokens: should be calculated as 977 - 768 = 209 + usage = Usage( + prompt_tokens=17, + completion_tokens=977, + total_tokens=994, + completion_tokens_details=CompletionTokensDetailsWrapper( + reasoning_tokens=768, + audio_tokens=0, + # text_tokens NOT provided - this is the key part of the bug + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + ) + + # gpt-5-nano pricing: $0.05/1M input, $0.40/1M output + expected_prompt_cost = 17 * 0.05 / 1_000_000 + expected_completion_cost = 977 * 0.40 / 1_000_000 # ALL tokens, not just reasoning + + assert abs(prompt_cost - expected_prompt_cost) < 1e-10, \ + f"Prompt cost incorrect: {prompt_cost} vs {expected_prompt_cost}" + + assert abs(completion_cost - expected_completion_cost) < 1e-10, \ + f"Completion cost incorrect: {completion_cost} vs {expected_completion_cost}" + + # Verify it's NOT using only reasoning_tokens (the bug) + wrong_cost = 768 * 0.40 / 1_000_000 # Only reasoning tokens + assert abs(completion_cost - wrong_cost) > 1e-6, \ + "Bug detected: Cost calculation is using only reasoning_tokens instead of all completion_tokens!" From 969790c4631f9efcbed0c55e0fb185ba98a0aa77 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Sun, 4 Jan 2026 00:10:07 +0530 Subject: [PATCH 211/388] Iam roles anywhere docs (#18559) * Add documentation for IAM Roles Anywhere Co-authored-by: krrishdholakia * Refactor Bedrock provider docs for IAM Roles Anywhere Co-authored-by: krrishdholakia --------- Co-authored-by: Cursor Agent --- docs/my-website/docs/providers/bedrock.md | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index 122554fe8a4..f1eed4b4d52 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -2208,6 +2208,53 @@ response = completion( | `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) | | `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) | +### IAM Roles Anywhere (On-Premise / External Workloads) + +[IAM Roles Anywhere](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/introduction.html) extends IAM roles to workloads **outside of AWS** (on-premise servers, edge devices, other clouds). It uses the same STS mechanism as regular IAM roles but authenticates via X.509 certificates instead of AWS credentials. + +**Setup**: Configure the [AWS Signing Helper](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/credential-helper.html) as a credential process in `~/.aws/config`: + +```ini +[profile litellm-roles-anywhere] +credential_process = aws_signing_helper credential-process \ + --certificate /path/to/certificate.pem \ + --private-key /path/to/private-key.pem \ + --trust-anchor-arn arn:aws:rolesanywhere:us-east-1:123456789012:trust-anchor/abc123 \ + --profile-arn arn:aws:rolesanywhere:us-east-1:123456789012:profile/def456 \ + --role-arn arn:aws:iam::123456789012:role/MyBedrockRole +``` + +**Usage**: Reference the profile in LiteLLM: + + + + +```python +from litellm import completion + +response = completion( + model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", + messages=[{"role": "user", "content": "Hello!"}], + aws_profile_name="litellm-roles-anywhere", +) +``` + + + + +```yaml +model_list: + - model_name: bedrock-claude + litellm_params: + model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 + aws_profile_name: "litellm-roles-anywhere" +``` + + + + +See the [IAM Roles Anywhere Getting Started Guide](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/getting-started.html) for trust anchor and profile setup. + Make the bedrock completion call From a3503e59c227f5f1dd15e9d8910f67a6b7dca2a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mehmet=20Can=20=C5=9Eakiro=C4=9Flu?= <53798389+cansakiroglu@users.noreply.github.com> Date: Sat, 3 Jan 2026 21:52:50 +0300 Subject: [PATCH 212/388] Litellm feat helm lifecycle support (#18517) * feat(helm): add lifecycle hook support for helm * add tests --- .../litellm-helm/templates/deployment.yaml | 4 ++++ .../litellm-helm/tests/deployment_tests.yaml | 24 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/deploy/charts/litellm-helm/templates/deployment.yaml b/deploy/charts/litellm-helm/templates/deployment.yaml index 0dab2ec40e0..19fa0479091 100644 --- a/deploy/charts/litellm-helm/templates/deployment.yaml +++ b/deploy/charts/litellm-helm/templates/deployment.yaml @@ -182,6 +182,10 @@ spec: {{- with .Values.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.lifecycle }} + lifecycle: + {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.extraContainers }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/deploy/charts/litellm-helm/tests/deployment_tests.yaml b/deploy/charts/litellm-helm/tests/deployment_tests.yaml index f9c83966696..182a2362392 100644 --- a/deploy/charts/litellm-helm/tests/deployment_tests.yaml +++ b/deploy/charts/litellm-helm/tests/deployment_tests.yaml @@ -136,4 +136,26 @@ tests: path: spec.template.spec.containers[0].volumeMounts content: name: litellm-config - mountPath: /etc/litellm/ \ No newline at end of file + mountPath: /etc/litellm/ + - it: should work with lifecycle hooks + template: deployment.yaml + set: + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - echo "Container stopping" + asserts: + - exists: + path: spec.template.spec.containers[0].lifecycle + - equal: + path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0] + value: /bin/sh + - equal: + path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1] + value: -c + - equal: + path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2] + value: echo "Container stopping" \ No newline at end of file From 3a4ebf173f637dc9aa1fdc037b27a3757e246851 Mon Sep 17 00:00:00 2001 From: Deepak Walia <58362408+dee-walia20@users.noreply.github.com> Date: Sun, 4 Jan 2026 00:35:53 +0530 Subject: [PATCH 213/388] fix(sap): honor allowed_openai_params in transform_request (#18432) --- litellm/llms/sap/chat/transformation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index 01ceb72c0de..e13abca59f8 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -203,9 +203,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): headers: dict, ) -> dict: supported_params = self.get_supported_openai_params(model) + # Include extra params that passed validation (e.g., thinking_config for Gemini models via allowed_openai_params) + extra_params = [k for k in optional_params if k not in supported_params and k not in {"tools", "model_version"}] + supported_params = supported_params + extra_params model_params = { k: v for k, v in optional_params.items() if k in supported_params } + model_version = optional_params.pop("model_version", "latest") template = [] for message in messages: From dc62cdb3009bff03a2282de1a981f29421e0383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=86=E3=82=8A?= Date: Sun, 4 Jan 2026 03:07:52 +0800 Subject: [PATCH 214/388] fix: handle empty error objects in response conversion (#18493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some OpenAI-compatible providers (e.g., Apertis) return empty error objects even on successful responses. The previous check only verified that error was not None, causing spurious APIErrors. Now the code checks if the error object contains meaningful data: - For dict errors: non-empty message OR non-null code - For string errors: non-empty string - Other truthy values are still treated as errors Fixes #18407 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: yurekami Co-authored-by: Claude Opus 4.5 --- .../convert_dict_to_response.py | 46 ++++-- .../test_convert_dict_to_chat_completion.py | 156 ++++++++++++++++++ 2 files changed, 188 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 59d2a8a8dd0..bbe28e3ec2c 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -445,25 +445,43 @@ def convert_to_model_response_object( # noqa: PLR0915 hidden_params["additional_headers"] = additional_headers ### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary + # Some OpenAI-compatible providers (e.g., Apertis) return empty error objects + # even on success. Only raise if the error contains meaningful data. if ( response_object is not None and "error" in response_object and response_object["error"] is not None ): - error_args = {"status_code": 422, "message": "Error in response object"} - if isinstance(response_object["error"], dict): - if "code" in response_object["error"]: - error_args["status_code"] = response_object["error"]["code"] - if "message" in response_object["error"]: - if isinstance(response_object["error"]["message"], dict): - message_str = json.dumps(response_object["error"]["message"]) - else: - message_str = str(response_object["error"]["message"]) - error_args["message"] = message_str - raised_exception = Exception() - setattr(raised_exception, "status_code", error_args["status_code"]) - setattr(raised_exception, "message", error_args["message"]) - raise raised_exception + error_obj = response_object["error"] + has_meaningful_error = False + + if isinstance(error_obj, dict): + # Check if error dict has non-empty message or non-null code + error_message = error_obj.get("message", "") + error_code = error_obj.get("code") + has_meaningful_error = bool(error_message) or error_code is not None + elif isinstance(error_obj, str): + # String error is meaningful if non-empty + has_meaningful_error = bool(error_obj) + else: + # Any other truthy value is considered meaningful + has_meaningful_error = True + + if has_meaningful_error: + error_args = {"status_code": 422, "message": "Error in response object"} + if isinstance(error_obj, dict): + if "code" in error_obj: + error_args["status_code"] = error_obj["code"] + if "message" in error_obj: + if isinstance(error_obj["message"], dict): + message_str = json.dumps(error_obj["message"]) + else: + message_str = str(error_obj["message"]) + error_args["message"] = message_str + raised_exception = Exception() + setattr(raised_exception, "status_code", error_args["status_code"]) + setattr(raised_exception, "message", error_args["message"]) + raise raised_exception try: if response_type == "completion" and ( diff --git a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py index 7e269f21451..c151150f634 100644 --- a/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py +++ b/tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py @@ -903,3 +903,159 @@ def test_convert_to_model_response_object_with_thinking_content(): resp: ModelResponse = convert_to_model_response_object(**args) assert resp is not None assert resp.choices[0].message.reasoning_content is not None + + +def test_convert_to_model_response_object_with_empty_error_object(): + """ + Test that convert_to_model_response_object handles empty error objects gracefully. + + This is a regression test for issue #18407 where providers like Apertis return + empty error objects even on successful responses, causing spurious APIErrors. + + The error object structure: + { + "error": { + "message": "", + "type": "", + "param": "", + "code": null + } + } + """ + response_object = { + "model": "minimax-m2.1", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hey! I'm doing well, thanks for asking!", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 49, + "completion_tokens": 87, + "total_tokens": 136, + }, + "error": { + "message": "", + "type": "", + "param": "", + "code": None, + }, + } + + # This should NOT raise an exception + result = convert_to_model_response_object( + model_response_object=ModelResponse(), + response_object=response_object, + stream=False, + start_time=datetime.now(), + end_time=datetime.now(), + hidden_params=None, + _response_headers=None, + convert_tool_call_to_json_mode=False, + ) + + assert isinstance(result, ModelResponse) + assert result.model == "minimax-m2.1" + assert len(result.choices) == 1 + assert result.choices[0].message.content == "Hey! I'm doing well, thanks for asking!" + + +def test_convert_to_model_response_object_with_real_error(): + """ + Test that convert_to_model_response_object still raises for real errors. + + Ensures the empty error fix doesn't break legitimate error handling. + """ + response_object = { + "error": { + "message": "Rate limit exceeded", + "type": "rate_limit_error", + "param": None, + "code": 429, + }, + } + + with pytest.raises(Exception) as exc_info: + convert_to_model_response_object( + model_response_object=ModelResponse(), + response_object=response_object, + stream=False, + start_time=datetime.now(), + end_time=datetime.now(), + hidden_params=None, + _response_headers=None, + convert_tool_call_to_json_mode=False, + ) + + # The exception should have the error message + assert hasattr(exc_info.value, "message") + assert "Rate limit exceeded" in str(exc_info.value.message) + + +def test_convert_to_model_response_object_with_empty_dict_error(): + """ + Test that convert_to_model_response_object handles completely empty error dict. + """ + response_object = { + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello!", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + "error": {}, # Completely empty error object + } + + # This should NOT raise an exception + result = convert_to_model_response_object( + model_response_object=ModelResponse(), + response_object=response_object, + stream=False, + start_time=datetime.now(), + end_time=datetime.now(), + hidden_params=None, + _response_headers=None, + convert_tool_call_to_json_mode=False, + ) + + assert isinstance(result, ModelResponse) + assert result.choices[0].message.content == "Hello!" + + +def test_convert_to_model_response_object_with_error_code_only(): + """ + Test that errors with only a code (no message) are still treated as real errors. + """ + response_object = { + "error": { + "message": "", + "code": 500, + }, + } + + with pytest.raises(Exception): + convert_to_model_response_object( + model_response_object=ModelResponse(), + response_object=response_object, + stream=False, + start_time=datetime.now(), + end_time=datetime.now(), + hidden_params=None, + _response_headers=None, + convert_tool_call_to_json_mode=False, + ) From 9ba27d85cee19e2cced4fbac4a2b68e7d7ae7dcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=82=86=E3=82=8A?= Date: Sun, 4 Jan 2026 03:08:32 +0800 Subject: [PATCH 215/388] feat(types): add output_text property to ResponsesAPIResponse (#18491) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the output_text convenience property to ResponsesAPIResponse that aggregates all output_text items from the output list, matching the OpenAI SDK's Response.output_text behavior. The property iterates through output items, collects text content from message-type outputs, and returns them concatenated into a single string. Returns empty string if no output_text content exists. Handles both dict and Pydantic model access patterns for compatibility with different output formats. Fixes #18470 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: yurekami Co-authored-by: Claude Opus 4.5 --- litellm/types/llms/openai.py | 33 +++++ .../types/llms/test_types_llms_openai.py | 134 ++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index ceeae958a80..c2912558cab 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1197,6 +1197,39 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): # Define private attributes using PrivateAttr _hidden_params: dict = PrivateAttr(default_factory=dict) + @property + def output_text(self) -> str: + """ + Convenience property that aggregates all `output_text` items from the `output` list. + + If no `output_text` content blocks exist, then an empty string is returned. + + This matches the OpenAI SDK's Response.output_text behavior. + """ + texts: List[str] = [] + for output_item in self.output: + # Handle both dict and object access patterns + if isinstance(output_item, dict): + item_type = output_item.get("type") + content = output_item.get("content", []) + else: + item_type = getattr(output_item, "type", None) + content = getattr(output_item, "content", []) + + if item_type == "message": + for content_item in content: + if isinstance(content_item, dict): + content_type = content_item.get("type") + text = content_item.get("text", "") + else: + content_type = getattr(content_item, "type", None) + text = getattr(content_item, "text", "") or "" + + if content_type == "output_text": + texts.append(text) + + return "".join(texts) + class ResponsesAPIStreamEvents(str, Enum): """ diff --git a/tests/test_litellm/types/llms/test_types_llms_openai.py b/tests/test_litellm/types/llms/test_types_llms_openai.py index 05dec06d469..87cc9586665 100644 --- a/tests/test_litellm/types/llms/test_types_llms_openai.py +++ b/tests/test_litellm/types/llms/test_types_llms_openai.py @@ -35,3 +35,137 @@ def test_output_item_added_event(): assert event.sequence_number == 4 assert event.output_index == 1 assert event.item is None + + +class TestResponsesAPIResponseOutputText: + """Tests for the output_text property on ResponsesAPIResponse""" + + def test_output_text_with_single_message(self): + """Test output_text with a single message containing text output""" + from litellm.types.llms.openai import ResponsesAPIResponse + + response = ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + output=[ + { + "type": "message", + "id": "msg_123", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Hello, world!", + } + ], + } + ], + ) + + assert response.output_text == "Hello, world!" + + def test_output_text_with_multiple_messages(self): + """Test output_text with multiple messages aggregates all text""" + from litellm.types.llms.openai import ResponsesAPIResponse + + response = ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "First part. ", + } + ], + }, + { + "type": "message", + "id": "msg_2", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "Second part.", + } + ], + }, + ], + ) + + assert response.output_text == "First part. Second part." + + def test_output_text_with_no_text_content(self): + """Test output_text returns empty string when no output_text content exists""" + from litellm.types.llms.openai import ResponsesAPIResponse + + response = ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + output=[ + { + "type": "function_call", + "id": "call_123", + "status": "completed", + "name": "get_weather", + "arguments": "{}", + } + ], + ) + + assert response.output_text == "" + + def test_output_text_with_mixed_content(self): + """Test output_text only aggregates output_text type content""" + from litellm.types.llms.openai import ResponsesAPIResponse + + response = ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "The weather is sunny. ", + }, + { + "type": "refusal", + "refusal": "I cannot do that.", + }, + ], + }, + { + "type": "function_call", + "id": "call_123", + "status": "completed", + "name": "get_weather", + "arguments": "{}", + }, + ], + ) + + assert response.output_text == "The weather is sunny. " + + def test_output_text_with_empty_output(self): + """Test output_text returns empty string with empty output list""" + from litellm.types.llms.openai import ResponsesAPIResponse + + response = ResponsesAPIResponse( + id="resp_123", + created_at=1234567890, + output=[], + ) + + assert response.output_text == "" From 37c908caf9777a757f813596ec04897aa995170f Mon Sep 17 00:00:00 2001 From: Lu Date: Sun, 4 Jan 2026 03:13:22 +0800 Subject: [PATCH 216/388] google genai adapter inline data support (#18477) * support inline data * add test --- .../google_genai/adapters/transformation.py | 62 +++++++-- .../google_genai/test_google_genai_adapter.py | 127 +++++++++++++++++- 2 files changed, 179 insertions(+), 10 deletions(-) diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py index 9d3f990b1aa..58a52666d38 100644 --- a/litellm/google_genai/adapters/transformation.py +++ b/litellm/google_genai/adapters/transformation.py @@ -8,8 +8,10 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAssistantMessage, ChatCompletionAssistantToolCall, + ChatCompletionImageObject, ChatCompletionRequest, ChatCompletionSystemMessage, + ChatCompletionTextObject, ChatCompletionToolCallFunctionChunk, ChatCompletionToolChoiceValues, ChatCompletionToolMessage, @@ -385,13 +387,36 @@ class GoogleGenAIAdapter: if role == "user": # Handle user messages with potential function responses - combined_text = "" + content_parts: List[ + Union[ChatCompletionTextObject, ChatCompletionImageObject] + ] = [] tool_messages: List[ChatCompletionToolMessage] = [] for part in parts: if isinstance(part, dict): if "text" in part: - combined_text += part["text"] + content_parts.append( + cast( + ChatCompletionTextObject, + {"type": "text", "text": part["text"]}, + ) + ) + elif "inline_data" in part: + # Handle Base64 image data + inline_data = part["inline_data"] + mime_type = inline_data.get("mime_type", "image/jpeg") + data = inline_data.get("data", "") + content_parts.append( + cast( + ChatCompletionImageObject, + { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{data}" + }, + }, + ) + ) elif "functionResponse" in part: # Transform function response to tool message func_response = part["functionResponse"] @@ -402,13 +427,33 @@ class GoogleGenAIAdapter: ) tool_messages.append(tool_message) elif isinstance(part, str): - combined_text += part + content_parts.append( + cast( + ChatCompletionTextObject, {"type": "text", "text": part} + ) + ) - # Add user message if there's text content - if combined_text: - messages.append( - ChatCompletionUserMessage(role="user", content=combined_text) - ) + # Add user message if there's content + if content_parts: + # If only one text part, use simple string format for backward compatibility + if ( + len(content_parts) == 1 + and isinstance(content_parts[0], dict) + and content_parts[0].get("type") == "text" + ): + text_part = cast(ChatCompletionTextObject, content_parts[0]) + messages.append( + ChatCompletionUserMessage( + role="user", content=text_part["text"] + ) + ) + else: + # Use multimodal format (array of content parts) + messages.append( + ChatCompletionUserMessage( + role="user", content=content_parts + ) + ) # Add tool messages messages.extend(tool_messages) @@ -468,7 +513,6 @@ class GoogleGenAIAdapter: Dict in Google GenAI generate_content response format """ - # Extract the main response content choice = response.choices[0] if response.choices else None if not choice: diff --git a/tests/test_litellm/google_genai/test_google_genai_adapter.py b/tests/test_litellm/google_genai/test_google_genai_adapter.py index e8882a1acb3..135881ad209 100644 --- a/tests/test_litellm/google_genai/test_google_genai_adapter.py +++ b/tests/test_litellm/google_genai/test_google_genai_adapter.py @@ -1197,6 +1197,131 @@ async def test_agenerate_content_x_goog_api_key_header(): # Verify other expected headers assert headers.get("Content-Type") == "application/json", f"Expected Content-Type application/json, got {headers.get('Content-Type')}" - + print(f"✓ Test passed: x-goog-api-key header correctly set to {api_key_value}") print(f"✓ All headers: {list(headers.keys())}") + + +def test_inline_data_base64_image_transformation(): + """Test transformation of Gemini inline_data (Base64 images) to OpenAI format""" + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter + + adapter = GoogleGenAIAdapter() + + # Test input with Base64 image + model = "gpt-4-vision-preview" + contents = { + "role": "user", + "parts": [ + {"text": "What's in this image?"}, + { + "inline_data": { + "mime_type": "image/jpeg", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + } + ] + } + + # Transform to completion format + completion_request = adapter.translate_generate_content_to_completion( + model=model, + contents=contents + ) + + # Verify the transformation + assert completion_request["model"] == model + assert len(completion_request["messages"]) == 1 + assert completion_request["messages"][0]["role"] == "user" + + # Verify content is an array (multimodal format) + content = completion_request["messages"][0]["content"] + assert isinstance(content, list), "Content should be a list for multimodal messages" + assert len(content) == 2, "Should have 2 content parts (text + image)" + + # Verify text part + text_part = content[0] + assert text_part["type"] == "text" + assert text_part["text"] == "What's in this image?" + + # Verify image part + image_part = content[1] + assert image_part["type"] == "image_url" + assert "image_url" in image_part + assert "url" in image_part["image_url"] + assert image_part["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" in image_part["image_url"]["url"] + + +def test_inline_data_image_only_transformation(): + """Test transformation of Gemini inline_data with only image (no text)""" + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter + + adapter = GoogleGenAIAdapter() + + # Test input with only Base64 image (no text) + model = "gpt-4-vision-preview" + contents = { + "role": "user", + "parts": [ + { + "inline_data": { + "mime_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + } + ] + } + + # Transform to completion format + completion_request = adapter.translate_generate_content_to_completion( + model=model, + contents=contents + ) + + # Verify the transformation + assert completion_request["model"] == model + assert len(completion_request["messages"]) == 1 + assert completion_request["messages"][0]["role"] == "user" + + # Verify content is an array (multimodal format) + content = completion_request["messages"][0]["content"] + assert isinstance(content, list), "Content should be a list for multimodal messages" + assert len(content) == 1, "Should have 1 content part (image only)" + + # Verify image part + image_part = content[0] + assert image_part["type"] == "image_url" + assert "image_url" in image_part + assert "url" in image_part["image_url"] + assert image_part["image_url"]["url"].startswith("data:image/png;base64,") + + +def test_inline_data_backward_compatibility_text_only(): + """Test that pure text messages still use simple string format (backward compatibility)""" + from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter + + adapter = GoogleGenAIAdapter() + + # Test input with only text (no images) + model = "gpt-3.5-turbo" + contents = { + "role": "user", + "parts": [{"text": "Hello, how are you?"}] + } + + # Transform to completion format + completion_request = adapter.translate_generate_content_to_completion( + model=model, + contents=contents + ) + + # Verify the transformation + assert completion_request["model"] == model + assert len(completion_request["messages"]) == 1 + assert completion_request["messages"][0]["role"] == "user" + + # Verify content is a simple string (not an array) for backward compatibility + content = completion_request["messages"][0]["content"] + assert isinstance(content, str), "Content should be a string for text-only messages (backward compatibility)" + assert content == "Hello, how are you?" From 9b1c5f7e360e7b448d9f16500e4ababba6f47b42 Mon Sep 17 00:00:00 2001 From: cantalupo555 Date: Sat, 3 Jan 2026 16:14:19 -0300 Subject: [PATCH 217/388] feat(zai): Add GLM-4.7 model with reasoning support (#18476) Add support for Z.AI GLM-4.7, latest flagship model with enhanced reasoning capabilities. Changes: - Add zai/glm-4.7 to model pricing with /bin/bash.60/M input, .20/M output - Add cached input pricing (/bin/bash.11/M) for GLM-4.7 - Add supports_reasoning flag to enable thinking parameter - Update ZAIChatConfig to support thinking parameter for models with reasoning - Update documentation with GLM-4.7 as latest flagship model - Add cached input column to pricing table (GLM-4.7 only) - Add tests for GLM-4.7 reasoning support and cost calculation - Update all examples to use GLM-4.7 Model specifications: - Context: 200K input, 128K output - Supports: reasoning, function calling, tool choice, prompt caching - Pricing: Same as GLM-4.6 with cache support See: https://docs.z.ai/guides/llm/glm-4.7 --- docs/my-website/docs/providers/zai.md | 36 +++++++++--------- litellm/llms/zai/chat/transformation.py | 11 +++++- ...odel_prices_and_context_window_backup.json | 14 +++++++ model_prices_and_context_window.json | 14 +++++++ .../llms/zai/test_zai_provider.py | 37 +++++++++++++++++++ 5 files changed, 94 insertions(+), 18 deletions(-) diff --git a/docs/my-website/docs/providers/zai.md b/docs/my-website/docs/providers/zai.md index 5055d0c1cdd..937ccd67680 100644 --- a/docs/my-website/docs/providers/zai.md +++ b/docs/my-website/docs/providers/zai.md @@ -19,7 +19,7 @@ import os os.environ['ZAI_API_KEY'] = "" response = completion( - model="zai/glm-4.6", + model="zai/glm-4.7", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -34,7 +34,7 @@ import os os.environ['ZAI_API_KEY'] = "" response = completion( - model="zai/glm-4.6", + model="zai/glm-4.7", messages=[ {"role": "user", "content": "hello from litellm"} ], @@ -51,7 +51,8 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet | Model Name | Function Call | Notes | |------------|---------------|-------| -| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context | +| glm-4.7 | `completion(model="zai/glm-4.7", messages)` | **Latest flagship**, 200K context, **Reasoning** | +| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | 200K context | | glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context | | glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model | | glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier | @@ -62,16 +63,17 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet ## Model Pricing -| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window | -|-------|---------------------|----------------------|----------------| -| glm-4.6 | $0.60 | $2.20 | 200K | -| glm-4.5 | $0.60 | $2.20 | 128K | -| glm-4.5v | $0.60 | $1.80 | 128K | -| glm-4.5-x | $2.20 | $8.90 | 128K | -| glm-4.5-air | $0.20 | $1.10 | 128K | -| glm-4.5-airx | $1.10 | $4.50 | 128K | -| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K | -| glm-4.5-flash | **FREE** | **FREE** | 128K | +| Model | Input ($/1M tokens) | Output ($/1M tokens) | Cached Input ($/1M tokens) | Context Window | +|-------|---------------------|----------------------|---------------------------|----------------| +| glm-4.7 | $0.60 | $2.20 | $0.11 | 200K | +| glm-4.6 | $0.60 | $2.20 | - | 200K | +| glm-4.5 | $0.60 | $2.20 | - | 128K | +| glm-4.5v | $0.60 | $1.80 | - | 128K | +| glm-4.5-x | $2.20 | $8.90 | - | 128K | +| glm-4.5-air | $0.20 | $1.10 | - | 128K | +| glm-4.5-airx | $1.10 | $4.50 | - | 128K | +| glm-4-32b-0414-128k | $0.10 | $0.10 | - | 128K | +| glm-4.5-flash | **FREE** | **FREE** | - | 128K | ## Using with LiteLLM Proxy @@ -84,7 +86,7 @@ import os os.environ['ZAI_API_KEY'] = "" response = completion( - model="zai/glm-4.6", + model="zai/glm-4.7", messages=[{"role": "user", "content": "Hello, how are you?"}], ) @@ -98,9 +100,9 @@ print(response.choices[0].message.content) ```yaml model_list: - - model_name: glm-4.6 + - model_name: glm-4.7 litellm_params: - model: zai/glm-4.6 + model: zai/glm-4.7 api_key: os.environ/ZAI_API_KEY - model_name: glm-4.5-flash # Free tier litellm_params: @@ -121,7 +123,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ -H 'Content-Type: application/json' \ -H 'Authorization: Bearer sk-1234' \ -d '{ - "model": "glm-4.6", + "model": "glm-4.7", "messages": [ { "role": "user", diff --git a/litellm/llms/zai/chat/transformation.py b/litellm/llms/zai/chat/transformation.py index 47b314d4e0d..4380256f0a4 100644 --- a/litellm/llms/zai/chat/transformation.py +++ b/litellm/llms/zai/chat/transformation.py @@ -20,7 +20,7 @@ class ZAIChatConfig(OpenAIGPTConfig): return api_base, dynamic_api_key def get_supported_openai_params(self, model: str) -> list: - return [ + base_params = [ "max_tokens", "stream", "stream_options", @@ -31,3 +31,12 @@ class ZAIChatConfig(OpenAIGPTConfig): "tool_choice", ] + import litellm + + try: + if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider): + base_params.append("thinking") + except Exception: + pass + + return base_params diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c585dac9063..d32adf54b5e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -29649,6 +29649,20 @@ "supports_vision": true, "supports_web_search": true }, + "zai/glm-4.7": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-4.6": { "input_cost_per_token": 6e-07, "output_cost_per_token": 2.2e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index df286e6540a..81b4469f24c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29691,6 +29691,20 @@ "supports_vision": true, "supports_web_search": true }, + "zai/glm-4.7": { + "cache_creation_input_token_cost": 0, + "cache_read_input_token_cost": 1.1e-07, + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.2e-06, + "litellm_provider": "zai", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "source": "https://docs.z.ai/guides/overview/pricing" + }, "zai/glm-4.6": { "input_cost_per_token": 6e-07, "output_cost_per_token": 2.2e-06, diff --git a/tests/test_litellm/llms/zai/test_zai_provider.py b/tests/test_litellm/llms/zai/test_zai_provider.py index a3d47d666bc..d1e4359d048 100644 --- a/tests/test_litellm/llms/zai/test_zai_provider.py +++ b/tests/test_litellm/llms/zai/test_zai_provider.py @@ -1,6 +1,7 @@ """ Tests for Z.AI (Zhipu AI) provider - GLM models """ + import json import math @@ -50,10 +51,12 @@ def test_zai_in_provider_lists(): def test_zai_models_in_model_cost(): """Test that ZAI models are in the model cost map""" import os + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") zai_models = [ + "zai/glm-4.7", "zai/glm-4.6", "zai/glm-4.5", "zai/glm-4.5v", @@ -72,6 +75,7 @@ def test_zai_models_in_model_cost(): def test_zai_glm46_cost_calculation(): """Test the cost calculation for glm-4.6""" import os + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -92,6 +96,7 @@ def test_zai_glm46_cost_calculation(): def test_zai_flash_model_is_free(): """Test that glm-4.5-flash has zero cost""" import os + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -102,6 +107,38 @@ def test_zai_flash_model_is_free(): assert info["output_cost_per_token"] == 0 +def test_glm47_supports_reasoning(): + """Test that GLM-4.7 supports reasoning""" + import os + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + key = "zai/glm-4.7" + assert key in litellm.model_cost, f"Model {key} not found in model_cost" + + info = litellm.model_cost[key] + assert info["supports_reasoning"] is True + + +def test_glm47_cost_calculation(): + """Test cost calculation for GLM-4.7""" + import os + + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + prompt_cost, completion_cost = cost_per_token( + model="zai/glm-4.7", + prompt_tokens=1000000, # 1M tokens + completion_tokens=1000000, + ) + + # GLM-4.7: $0.6/M input, $2.2/M output (same as GLM-4.6) + assert math.isclose(prompt_cost, 0.6, rel_tol=1e-6) + assert math.isclose(completion_cost, 2.2, rel_tol=1e-6) + + @pytest.mark.asyncio async def test_zai_completion_call(respx_mock, zai_response, monkeypatch): """Test completion call with zai provider using mocked response""" From 89b4a6d67c2e603e294bcf2f35bf34bd7e9ba2ae Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Sat, 3 Jan 2026 11:15:15 -0800 Subject: [PATCH 218/388] Allow installation with current grpcio on old Python (#18473) Instead of limiting grpcio < 1.68.0, specifically exclude the versions affected by the reconnect bug, and allow installation with either older or newer versions. Signed-off-by: Anders Kaseorg --- poetry.lock | 72 +----------------------------------------------- pyproject.toml | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 73 deletions(-) diff --git a/poetry.lock b/poetry.lock index ee97c00594c..a0a0f8540e5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2273,75 +2273,6 @@ googleapis-common-protos = {version = ">=1.56.0,<2.0.0", extras = ["grpc"]} grpcio = ">=1.44.0,<2.0.0" protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" -[[package]] -name = "grpcio" -version = "1.67.1" -description = "HTTP/2-based RPC framework" -optional = false -python-versions = ">=3.8" -groups = ["main", "dev", "proxy-dev"] -markers = "python_version < \"3.14\"" -files = [ - {file = "grpcio-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:8b0341d66a57f8a3119b77ab32207072be60c9bf79760fa609c5609f2deb1f3f"}, - {file = "grpcio-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5a27dddefe0e2357d3e617b9079b4bfdc91341a91565111a21ed6ebbc51b22d"}, - {file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:43112046864317498a33bdc4797ae6a268c36345a910de9b9c17159d8346602f"}, - {file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9b929f13677b10f63124c1a410994a401cdd85214ad83ab67cc077fc7e480f0"}, - {file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7d1797a8a3845437d327145959a2c0c47c05947c9eef5ff1a4c80e499dcc6fa"}, - {file = "grpcio-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0489063974d1452436139501bf6b180f63d4977223ee87488fe36858c5725292"}, - {file = "grpcio-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9fd042de4a82e3e7aca44008ee2fb5da01b3e5adb316348c21980f7f58adc311"}, - {file = "grpcio-1.67.1-cp310-cp310-win32.whl", hash = "sha256:638354e698fd0c6c76b04540a850bf1db27b4d2515a19fcd5cf645c48d3eb1ed"}, - {file = "grpcio-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:608d87d1bdabf9e2868b12338cd38a79969eaf920c89d698ead08f48de9c0f9e"}, - {file = "grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb"}, - {file = "grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e"}, - {file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f"}, - {file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc"}, - {file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96"}, - {file = "grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f"}, - {file = "grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970"}, - {file = "grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744"}, - {file = "grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5"}, - {file = "grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953"}, - {file = "grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af"}, - {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e"}, - {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75"}, - {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38"}, - {file = "grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78"}, - {file = "grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc"}, - {file = "grpcio-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa0162e56fd10a5547fac8774c4899fc3e18c1aa4a4759d0ce2cd00d3696ea6b"}, - {file = "grpcio-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:beee96c8c0b1a75d556fe57b92b58b4347c77a65781ee2ac749d550f2a365dc1"}, - {file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:a93deda571a1bf94ec1f6fcda2872dad3ae538700d94dc283c672a3b508ba3af"}, - {file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e6f255980afef598a9e64a24efce87b625e3e3c80a45162d111a461a9f92955"}, - {file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e838cad2176ebd5d4a8bb03955138d6589ce9e2ce5d51c3ada34396dbd2dba8"}, - {file = "grpcio-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a6703916c43b1d468d0756c8077b12017a9fcb6a1ef13faf49e67d20d7ebda62"}, - {file = "grpcio-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:917e8d8994eed1d86b907ba2a61b9f0aef27a2155bca6cbb322430fc7135b7bb"}, - {file = "grpcio-1.67.1-cp313-cp313-win32.whl", hash = "sha256:e279330bef1744040db8fc432becc8a727b84f456ab62b744d3fdb83f327e121"}, - {file = "grpcio-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:fa0c739ad8b1996bd24823950e3cb5152ae91fca1c09cc791190bf1627ffefba"}, - {file = "grpcio-1.67.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:178f5db771c4f9a9facb2ab37a434c46cb9be1a75e820f187ee3d1e7805c4f65"}, - {file = "grpcio-1.67.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0f3e49c738396e93b7ba9016e153eb09e0778e776df6090c1b8c91877cc1c426"}, - {file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:24e8a26dbfc5274d7474c27759b54486b8de23c709d76695237515bc8b5baeab"}, - {file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b6c16489326d79ead41689c4b84bc40d522c9a7617219f4ad94bc7f448c5085"}, - {file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e6a4dcf5af7bbc36fd9f81c9f372e8ae580870a9e4b6eafe948cd334b81cf3"}, - {file = "grpcio-1.67.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:95b5f2b857856ed78d72da93cd7d09b6db8ef30102e5e7fe0961fe4d9f7d48e8"}, - {file = "grpcio-1.67.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b49359977c6ec9f5d0573ea4e0071ad278ef905aa74e420acc73fd28ce39e9ce"}, - {file = "grpcio-1.67.1-cp38-cp38-win32.whl", hash = "sha256:f5b76ff64aaac53fede0cc93abf57894ab2a7362986ba22243d06218b93efe46"}, - {file = "grpcio-1.67.1-cp38-cp38-win_amd64.whl", hash = "sha256:804c6457c3cd3ec04fe6006c739579b8d35c86ae3298ffca8de57b493524b771"}, - {file = "grpcio-1.67.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:a25bdea92b13ff4d7790962190bf6bf5c4639876e01c0f3dda70fc2769616335"}, - {file = "grpcio-1.67.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cdc491ae35a13535fd9196acb5afe1af37c8237df2e54427be3eecda3653127e"}, - {file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:85f862069b86a305497e74d0dc43c02de3d1d184fc2c180993aa8aa86fbd19b8"}, - {file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec74ef02010186185de82cc594058a3ccd8d86821842bbac9873fd4a2cf8be8d"}, - {file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01f616a964e540638af5130469451cf580ba8c7329f45ca998ab66e0c7dcdb04"}, - {file = "grpcio-1.67.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:299b3d8c4f790c6bcca485f9963b4846dd92cf6f1b65d3697145d005c80f9fe8"}, - {file = "grpcio-1.67.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:60336bff760fbb47d7e86165408126f1dded184448e9a4c892189eb7c9d3f90f"}, - {file = "grpcio-1.67.1-cp39-cp39-win32.whl", hash = "sha256:5ed601c4c6008429e3d247ddb367fe8c7259c355757448d7c1ef7bd4a6739e8e"}, - {file = "grpcio-1.67.1-cp39-cp39-win_amd64.whl", hash = "sha256:5db70d32d6703b89912af16d6d45d78406374a8b8ef0d28140351dd0ec610e98"}, - {file = "grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732"}, -] - -[package.extras] -protobuf = ["grpcio-tools (>=1.67.1)"] - [[package]] name = "grpcio" version = "1.76.0" @@ -2349,7 +2280,6 @@ description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.9" groups = ["main", "dev", "proxy-dev"] -markers = "python_version >= \"3.14\"" files = [ {file = "grpcio-1.76.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc"}, {file = "grpcio-1.76.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde"}, @@ -8051,4 +7981,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "b010d9da7f5a765670932b78d720aae4fcb819daba050683ee125b4367972419" +content-hash = "7eed2b2c25173a275ac83c55fd901b9b84663b1d7daa54f0e78b30bf1c8f0e3e" diff --git a/pyproject.toml b/pyproject.toml index f929fb94cb0..3b09119a748 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,7 +72,7 @@ soundfile = {version = "^0.12.1", optional = true} # - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290) # - 1.75.0+ has Python 3.14 wheels and bug fix grpcio = [ - {version = ">=1.62.3,<1.68.0", python = "<3.14"}, + {version = ">=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!=1.73.0", python = "<3.14"}, {version = ">=1.75.0", python = ">=3.14"}, ] diff --git a/requirements.txt b/requirements.txt index 3bc968c8cb8..06a7c17336c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ opentelemetry-api==1.25.0 opentelemetry-sdk==1.25.0 opentelemetry-exporter-otlp==1.25.0 # grpcio: 1.68.0-1.68.1 has reconnect bug (#38290), 1.75+ has Python 3.14 wheels + fix -grpcio>=1.62.3,<1.68.0; python_version < "3.14" +grpcio>=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!=1.73.0; python_version < "3.14" grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests From 099e108b51df00d2c8e855b459264973b1bb03b0 Mon Sep 17 00:00:00 2001 From: lif <1835304752@qq.com> Date: Sun, 4 Jan 2026 03:15:45 +0800 Subject: [PATCH 219/388] fix: correctly route codestral chat and FIM endpoints (#18467) Fixed duplicate condition that made text-completion-codestral provider unreachable. Now: - codestral.mistral.ai/v1/chat/completions -> codestral - codestral.mistral.ai/v1/fim/completions -> text-completion-codestral Fixes #18464 Signed-off-by: majiayu000 <1835304752@qq.com> --- .../get_llm_provider_logic.py | 4 +- .../test_codestral_provider_routing.py | 69 +++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/test_codestral_provider_routing.py diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 164e2a73e65..b753e9fa8b5 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -229,10 +229,10 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.ai21.com/studio/v1": custom_llm_provider = "ai21_chat" dynamic_api_key = get_secret_str("AI21_API_KEY") - elif endpoint == "https://codestral.mistral.ai/v1": + elif endpoint == "codestral.mistral.ai/v1/chat/completions": custom_llm_provider = "codestral" dynamic_api_key = get_secret_str("CODESTRAL_API_KEY") - elif endpoint == "https://codestral.mistral.ai/v1": + elif endpoint == "codestral.mistral.ai/v1/fim/completions": custom_llm_provider = "text-completion-codestral" dynamic_api_key = get_secret_str("CODESTRAL_API_KEY") elif endpoint == "app.empower.dev/api/v1": diff --git a/tests/test_litellm/litellm_core_utils/test_codestral_provider_routing.py b/tests/test_litellm/litellm_core_utils/test_codestral_provider_routing.py new file mode 100644 index 00000000000..1a6ed51afd0 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_codestral_provider_routing.py @@ -0,0 +1,69 @@ +""" +Unit tests for codestral provider routing. + +These tests verify that the chat and FIM endpoints for codestral +are correctly routed to different providers: +- Chat endpoint -> codestral provider +- FIM endpoint -> text-completion-codestral provider + +Related issue: https://github.com/BerriAI/litellm/issues/18464 +""" +import pytest + +import litellm + + +class TestCodestralProviderRouting: + """Tests for codestral endpoint routing in get_llm_provider""" + + def test_codestral_chat_endpoint_routes_to_codestral_provider(self): + """ + Test that the codestral chat endpoint routes to the 'codestral' provider. + + The chat/completions endpoint should be handled by the codestral provider. + """ + model, custom_llm_provider, _, api_base = litellm.get_llm_provider( + model="codestral-latest", + api_base="https://codestral.mistral.ai/v1/chat/completions", + ) + + assert custom_llm_provider == "codestral" + + def test_codestral_fim_endpoint_routes_to_text_completion_provider(self): + """ + Test that the codestral FIM endpoint routes to 'text-completion-codestral'. + + The fim/completions endpoint should be handled by the + text-completion-codestral provider for fill-in-the-middle completions. + """ + model, custom_llm_provider, _, api_base = litellm.get_llm_provider( + model="codestral-latest", + api_base="https://codestral.mistral.ai/v1/fim/completions", + ) + + assert custom_llm_provider == "text-completion-codestral" + + def test_codestral_endpoints_are_different_providers(self): + """ + Test that chat and FIM endpoints route to different providers. + + This is the core fix for issue #18464 - previously both endpoints + would route to 'codestral' due to duplicate conditions. + """ + _, chat_provider, _, _ = litellm.get_llm_provider( + model="codestral-latest", + api_base="https://codestral.mistral.ai/v1/chat/completions", + ) + + _, fim_provider, _, _ = litellm.get_llm_provider( + model="codestral-latest", + api_base="https://codestral.mistral.ai/v1/fim/completions", + ) + + assert chat_provider != fim_provider + assert chat_provider == "codestral" + assert fim_provider == "text-completion-codestral" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 64cfe75bfd398aa856c117aedfb6b72c70f48d16 Mon Sep 17 00:00:00 2001 From: lif <1835304752@qq.com> Date: Sun, 4 Jan 2026 03:17:38 +0800 Subject: [PATCH 220/388] fix: extract pure base64 data from data URLs for Ollama (#18465) Fix Ollama_chatException "illegal base64 data at input byte 4" error when using images with ollama_chat provider. Ollama expects pure base64 data, not the full data URL format (data:image/png;base64,...). Fixes #18338 Signed-off-by: majiayu000 <1835304752@qq.com> --- .../prompt_templates/common_utils.py | 32 +++- .../test_extract_base64_image.py | 156 ++++++++++++++++++ 2 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/test_extract_base64_image.py diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index ca2a092dbc8..b100b9b516b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1087,9 +1087,35 @@ def _parse_content_for_reasoning( return None, message_text +def _extract_base64_data(image_url: str) -> str: + """ + Extract pure base64 data from an image URL. + + If the URL is a data URL (e.g., "data:image/png;base64,iVBOR..."), + extract and return only the base64 data portion. + Otherwise, return the original URL unchanged. + + This is needed for providers like Ollama that expect pure base64 data + rather than full data URLs. + + Args: + image_url: The image URL or data URL to process + + Returns: + The base64 data if it's a data URL, otherwise the original URL + """ + if image_url.startswith("data:") and ";base64," in image_url: + return image_url.split(";base64,", 1)[1] + return image_url + + def extract_images_from_message(message: AllMessageValues) -> List[str]: """ - Extract images from a message + Extract images from a message. + + For data URLs (e.g., "data:image/png;base64,iVBOR..."), only the base64 + data portion is extracted. This is required for providers like Ollama + that expect pure base64 data rather than full data URLs. """ images = [] message_content = message.get("content") @@ -1098,7 +1124,7 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]: image_url = m.get("image_url") if image_url: if isinstance(image_url, str): - images.append(image_url) + images.append(_extract_base64_data(image_url)) elif isinstance(image_url, dict) and "url" in image_url: - images.append(image_url["url"]) + images.append(_extract_base64_data(image_url["url"])) return images diff --git a/tests/test_litellm/litellm_core_utils/test_extract_base64_image.py b/tests/test_litellm/litellm_core_utils/test_extract_base64_image.py new file mode 100644 index 00000000000..b17c02d7006 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/test_extract_base64_image.py @@ -0,0 +1,156 @@ +""" +Unit tests for _extract_base64_data and extract_images_from_message functions. + +These tests verify that base64 image data is correctly extracted from data URLs, +which fixes the Ollama error "illegal base64 data at input byte 4". + +Related issue: https://github.com/BerriAI/litellm/issues/18338 +""" +import pytest + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _extract_base64_data, + extract_images_from_message, +) + + +class TestExtractBase64Data: + """Tests for _extract_base64_data function""" + + def test_extract_base64_from_png_data_url(self): + """Test extracting base64 data from a PNG data URL""" + data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk" + expected = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk" + assert _extract_base64_data(data_url) == expected + + def test_extract_base64_from_jpeg_data_url(self): + """Test extracting base64 data from a JPEG data URL""" + data_url = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD" + expected = "/9j/4AAQSkZJRgABAQAAAQABAAD" + assert _extract_base64_data(data_url) == expected + + def test_extract_base64_from_gif_data_url(self): + """Test extracting base64 data from a GIF data URL""" + data_url = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP" + expected = "R0lGODlhAQABAIAAAAAAAP" + assert _extract_base64_data(data_url) == expected + + def test_regular_url_unchanged(self): + """Test that regular HTTP URLs are returned unchanged""" + url = "https://example.com/image.png" + assert _extract_base64_data(url) == url + + def test_file_path_unchanged(self): + """Test that file paths are returned unchanged""" + path = "/path/to/image.png" + assert _extract_base64_data(path) == path + + def test_data_url_without_base64_unchanged(self): + """Test that data URLs without base64 encoding are returned unchanged""" + # This is a data URL with URL encoding, not base64 + url = "data:text/plain,Hello%20World" + assert _extract_base64_data(url) == url + + def test_base64_data_with_special_chars(self): + """Test extracting base64 data that contains valid special characters""" + # Base64 can contain +, /, and = characters + data_url = "data:image/png;base64,abc+def/ghi===" + expected = "abc+def/ghi===" + assert _extract_base64_data(data_url) == expected + + +class TestExtractImagesFromMessage: + """Tests for extract_images_from_message function""" + + def test_extract_from_message_with_data_url_string(self): + """Test extracting images when image_url is a string data URL""" + message = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": "data:image/png;base64,iVBORw0KGgo", + } + ], + } + result = extract_images_from_message(message) + assert result == ["iVBORw0KGgo"] + + def test_extract_from_message_with_data_url_dict(self): + """Test extracting images when image_url is a dict with url key""" + message = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,iVBORw0KGgo"}, + } + ], + } + result = extract_images_from_message(message) + assert result == ["iVBORw0KGgo"] + + def test_extract_from_message_with_regular_url(self): + """Test that regular URLs are preserved""" + message = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + } + ], + } + result = extract_images_from_message(message) + assert result == ["https://example.com/image.png"] + + def test_extract_multiple_images(self): + """Test extracting multiple images from a single message""" + message = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": "data:image/png;base64,image1base64", + }, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,image2base64"}, + }, + { + "type": "image_url", + "image_url": "https://example.com/image3.png", + }, + ], + } + result = extract_images_from_message(message) + assert result == [ + "image1base64", + "image2base64", + "https://example.com/image3.png", + ] + + def test_empty_content(self): + """Test message with empty content""" + message = {"role": "user", "content": []} + result = extract_images_from_message(message) + assert result == [] + + def test_no_images_in_content(self): + """Test message with content but no images""" + message = { + "role": "user", + "content": [{"type": "text", "text": "Hello world"}], + } + result = extract_images_from_message(message) + assert result == [] + + def test_string_content(self): + """Test message with string content (no images possible)""" + message = {"role": "user", "content": "Hello world"} + result = extract_images_from_message(message) + assert result == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 7d81d245fb2665f279c69da1a2066532b07f3045 Mon Sep 17 00:00:00 2001 From: lif <1835304752@qq.com> Date: Sun, 4 Jan 2026 03:18:09 +0800 Subject: [PATCH 221/388] fix: align prometheus metric names with DEFINED_PROMETHEUS_METRICS (#18463) Fix metric name inconsistency for litellm_remaining_requests_metric and litellm_remaining_tokens_metric. The factory received names without the _metric suffix, causing _is_metric_enabled to fail when users configured these metrics in prometheus_metrics_config. Fixes #18221 Signed-off-by: majiayu000 <1835304752@qq.com> --- litellm/integrations/prometheus.py | 4 +- ...test_prometheus_metric_name_consistency.py | 106 ++++++++++++++++++ 2 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 20f1357a1c8..c01f7481277 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -214,7 +214,7 @@ class PrometheusLogger(CustomLogger): # Remaining Rate Limit for model self.litellm_remaining_requests_metric = self._gauge_factory( - "litellm_remaining_requests", + "litellm_remaining_requests_metric", "LLM Deployment Analytics - remaining requests for model, returned from LLM API Provider", labelnames=self.get_labels_for_metric( "litellm_remaining_requests_metric" @@ -222,7 +222,7 @@ class PrometheusLogger(CustomLogger): ) self.litellm_remaining_tokens_metric = self._gauge_factory( - "litellm_remaining_tokens", + "litellm_remaining_tokens_metric", "remaining tokens for model, returned from LLM API Provider", labelnames=self.get_labels_for_metric( "litellm_remaining_tokens_metric" diff --git a/tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py b/tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py new file mode 100644 index 00000000000..9658eff3cc5 --- /dev/null +++ b/tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py @@ -0,0 +1,106 @@ +""" +Unit tests for prometheus metric name consistency + +This test ensures that the metric names used when creating Prometheus metrics +match the names defined in DEFINED_PROMETHEUS_METRICS, so that metric filtering +configuration works correctly. + +Related issue: https://github.com/BerriAI/litellm/issues/18221 +""" +from typing import get_args + +import pytest + + +def test_remaining_requests_metric_name_in_defined_metrics(): + """ + Test that litellm_remaining_requests_metric is defined in DEFINED_PROMETHEUS_METRICS. + + The metric name should include the _metric suffix to be consistent with the + configuration format users specify in prometheus_metrics_config. + """ + from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS + + defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS) + assert ( + "litellm_remaining_requests_metric" in defined_metrics + ), "litellm_remaining_requests_metric should be in DEFINED_PROMETHEUS_METRICS" + + +def test_remaining_tokens_metric_name_in_defined_metrics(): + """ + Test that litellm_remaining_tokens_metric is defined in DEFINED_PROMETHEUS_METRICS. + + The metric name should include the _metric suffix to be consistent with the + configuration format users specify in prometheus_metrics_config. + """ + from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS + + defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS) + assert ( + "litellm_remaining_tokens_metric" in defined_metrics + ), "litellm_remaining_tokens_metric should be in DEFINED_PROMETHEUS_METRICS" + + +def test_prometheus_metric_labels_have_remaining_metrics(): + """ + Test that PrometheusMetricLabels has label definitions for remaining metrics. + + This ensures that the labels can be retrieved when creating the metrics. + """ + from litellm.types.integrations.prometheus import PrometheusMetricLabels + + # Test that labels can be retrieved for remaining metrics + remaining_requests_labels = PrometheusMetricLabels.get_labels( + "litellm_remaining_requests_metric" + ) + remaining_tokens_labels = PrometheusMetricLabels.get_labels( + "litellm_remaining_tokens_metric" + ) + + assert isinstance( + remaining_requests_labels, list + ), "Labels for litellm_remaining_requests_metric should be a list" + assert isinstance( + remaining_tokens_labels, list + ), "Labels for litellm_remaining_tokens_metric should be a list" + + # These metrics should have api_provider and api_base labels + assert ( + "api_provider" in remaining_requests_labels + ), "litellm_remaining_requests_metric should have api_provider label" + assert ( + "api_base" in remaining_requests_labels + ), "litellm_remaining_requests_metric should have api_base label" + assert ( + "api_provider" in remaining_tokens_labels + ), "litellm_remaining_tokens_metric should have api_provider label" + assert ( + "api_base" in remaining_tokens_labels + ), "litellm_remaining_tokens_metric should have api_base label" + + +def test_all_defined_metrics_have_consistent_naming(): + """ + Test that all metrics defined in DEFINED_PROMETHEUS_METRICS follow + a consistent naming convention. + + This helps prevent similar inconsistencies in the future. + """ + from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS + + defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS) + + for metric_name in defined_metrics: + # All metrics should start with 'litellm_' + assert metric_name.startswith( + "litellm_" + ), f"Metric {metric_name} should start with 'litellm_'" + + +if __name__ == "__main__": + test_remaining_requests_metric_name_in_defined_metrics() + test_remaining_tokens_metric_name_in_defined_metrics() + test_prometheus_metric_labels_have_remaining_metrics() + test_all_defined_metrics_have_consistent_naming() + print("All prometheus metric name consistency tests passed!") From 1452f0150551193bd26227a348f2e0acd6d294fd Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 12:12:24 -0800 Subject: [PATCH 222/388] refactor: lazy load get_llm_provider and remove_index_from_tool_calls (#18608) --- litellm/_lazy_imports_registry.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 1a54e77998a..93fa8b39af2 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -32,6 +32,7 @@ UTILS_NAMES = ( "ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse", "TranscriptionResponse", "TextCompletionResponse", "get_provider_fields", "ModelResponseListIterator", "get_valid_models", "timeout", + "get_llm_provider", "remove_index_from_tool_calls", ) # Token counter names that support lazy loading via _lazy_import_token_counter @@ -336,6 +337,8 @@ _UTILS_IMPORT_MAP = { "ModelResponseListIterator": (".utils", "ModelResponseListIterator"), "get_valid_models": (".utils", "get_valid_models"), "timeout": (".timeout", "timeout"), + "get_llm_provider": ("litellm.litellm_core_utils.get_llm_provider_logic", "get_llm_provider"), + "remove_index_from_tool_calls": ("litellm.litellm_core_utils.core_helpers", "remove_index_from_tool_calls"), } _COST_CALCULATOR_IMPORT_MAP = { From 4904ed394eeace50bf951c8a0c9b2fd49b77b2c1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 12:36:26 -0800 Subject: [PATCH 223/388] Edit path for SSO Settings --- .../hooks/sso/useEditSSOSettings.ts | 38 +++ .../(dashboard)/hooks/sso/useSSOSettings.ts | 9 +- .../Modals/AddSSOSettingsModal.test.tsx | 2 +- .../Modals/AddSSOSettingsModal.tsx | 245 +++------------- .../Modals/BaseSSOSettingsForm.tsx | 249 ++++++++++++++++ .../Modals/EditSSOSettingsModal.tsx | 131 +++++++++ .../SSOSettings/RedactableField.tsx | 38 +++ .../AdminSettings/SSOSettings/SSOSettings.tsx | 84 ++++-- .../AdminSettings/SSOSettings/constants.ts | 15 + .../AdminSettings/SSOSettings/utils.test.ts | 274 ++++++++++++++++++ .../AdminSettings/SSOSettings/utils.ts | 54 ++++ 11 files changed, 894 insertions(+), 245 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useEditSSOSettings.ts create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useEditSSOSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useEditSSOSettings.ts new file mode 100644 index 00000000000..69e52d0ff25 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useEditSSOSettings.ts @@ -0,0 +1,38 @@ +import { useMutation, UseMutationResult } from "@tanstack/react-query"; +import { updateSSOSettings } from "@/components/networking"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; + +export interface EditSSOSettingsParams { + google_client_id?: string | null; + google_client_secret?: string | null; + microsoft_client_id?: string | null; + microsoft_client_secret?: string | null; + microsoft_tenant?: string | null; + generic_client_id?: string | null; + generic_client_secret?: string | null; + generic_authorization_endpoint?: string | null; + generic_token_endpoint?: string | null; + generic_userinfo_endpoint?: string | null; + proxy_base_url?: string | null; + user_email?: string | null; + sso_provider?: string | null; + role_mappings?: any; + [key: string]: any; +} + +export interface EditSSOSettingsResponse { + [key: string]: any; +} + +export const useEditSSOSettings = (): UseMutationResult => { + const { accessToken } = useAuthorized(); + + return useMutation({ + mutationFn: async (params: EditSSOSettingsParams) => { + if (!accessToken) { + throw new Error("Access token is required"); + } + return await updateSSOSettings(accessToken, params); + }, + }); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts index 6453b35ff93..3e09c3c2ca8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/sso/useSSOSettings.ts @@ -27,7 +27,14 @@ export interface SSOSettingsValues { proxy_base_url: string | null; user_email: string | null; ui_access_mode: string | null; - role_mappings: string | null; + role_mappings: { + provider: string; + group_claim: string; + default_role: "internal_user" | "internal_user_viewer" | "proxy_admin" | "proxy_admin_viewer"; + roles: { + [key: string]: string[]; + }; + }; } export interface SSOSettingsResponse { diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx index 13363a11643..aae28191031 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx @@ -17,7 +17,7 @@ describe("AddSSOSettingsModal", () => { const onCancel = vi.fn(); const onSuccess = vi.fn(); - render(); + render(); expect(screen.getByText("SSO Provider")).toBeInTheDocument(); expect(screen.getByText("Cancel")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.tsx index a4ef2938f60..7af6240b19e 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.tsx @@ -1,145 +1,36 @@ "use client"; import NotificationsManager from "@/components/molecules/notifications_manager"; -import { updateSSOSettings } from "@/components/networking"; import { parseErrorMessage } from "@/components/shared/errorUtils"; -import { TextInput } from "@tremor/react"; -import { Button as Button2, Form, Input, Modal, Select } from "antd"; -import React, { useState } from "react"; +import { Button, Form, Modal, Space } from "antd"; +import React from "react"; +import BaseSSOSettingsForm from "./BaseSSOSettingsForm"; +import { useEditSSOSettings } from "@/app/(dashboard)/hooks/sso/useEditSSOSettings"; +import { processSSOSettingsPayload } from "../utils"; interface AddSSOSettingsModalProps { isVisible: boolean; onCancel: () => void; onSuccess: () => void; - accessToken: string | null; } -const ssoProviderLogoMap: Record = { - google: "https://artificialanalysis.ai/img/logos/google_small.svg", - microsoft: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", - okta: "https://www.okta.com/sites/default/files/Okta_Logo_BrightBlue_Medium.png", - generic: "", -}; - -// Define the SSO provider configuration type -interface SSOProviderConfig { - envVarMap: Record; - fields: Array<{ - label: string; - name: string; - placeholder?: string; - }>; -} - -// Define configurations for each SSO provider -const ssoProviderConfigs: Record = { - google: { - envVarMap: { - google_client_id: "GOOGLE_CLIENT_ID", - google_client_secret: "GOOGLE_CLIENT_SECRET", - }, - fields: [ - { label: "Google Client ID", name: "google_client_id" }, - { label: "Google Client Secret", name: "google_client_secret" }, - ], - }, - microsoft: { - envVarMap: { - microsoft_client_id: "MICROSOFT_CLIENT_ID", - microsoft_client_secret: "MICROSOFT_CLIENT_SECRET", - microsoft_tenant: "MICROSOFT_TENANT", - }, - fields: [ - { label: "Microsoft Client ID", name: "microsoft_client_id" }, - { label: "Microsoft Client Secret", name: "microsoft_client_secret" }, - { label: "Microsoft Tenant", name: "microsoft_tenant" }, - ], - }, - okta: { - envVarMap: { - generic_client_id: "GENERIC_CLIENT_ID", - generic_client_secret: "GENERIC_CLIENT_SECRET", - generic_authorization_endpoint: "GENERIC_AUTHORIZATION_ENDPOINT", - generic_token_endpoint: "GENERIC_TOKEN_ENDPOINT", - generic_userinfo_endpoint: "GENERIC_USERINFO_ENDPOINT", - }, - fields: [ - { label: "Generic Client ID", name: "generic_client_id" }, - { label: "Generic Client Secret", name: "generic_client_secret" }, - { - label: "Authorization Endpoint", - name: "generic_authorization_endpoint", - placeholder: "https://your-domain/authorize", - }, - { label: "Token Endpoint", name: "generic_token_endpoint", placeholder: "https://your-domain/token" }, - { - label: "Userinfo Endpoint", - name: "generic_userinfo_endpoint", - placeholder: "https://your-domain/userinfo", - }, - ], - }, - generic: { - envVarMap: { - generic_client_id: "GENERIC_CLIENT_ID", - generic_client_secret: "GENERIC_CLIENT_SECRET", - generic_authorization_endpoint: "GENERIC_AUTHORIZATION_ENDPOINT", - generic_token_endpoint: "GENERIC_TOKEN_ENDPOINT", - generic_userinfo_endpoint: "GENERIC_USERINFO_ENDPOINT", - }, - fields: [ - { label: "Generic Client ID", name: "generic_client_id" }, - { label: "Generic Client Secret", name: "generic_client_secret" }, - { label: "Authorization Endpoint", name: "generic_authorization_endpoint" }, - { label: "Token Endpoint", name: "generic_token_endpoint" }, - { label: "Userinfo Endpoint", name: "generic_userinfo_endpoint" }, - ], - }, -}; - -const AddSSOSettingsModal: React.FC = ({ isVisible, onCancel, onSuccess, accessToken }) => { +const AddSSOSettingsModal: React.FC = ({ isVisible, onCancel, onSuccess }) => { const [form] = Form.useForm(); - const [isSubmitting, setIsSubmitting] = useState(false); + const { mutateAsync, isPending } = useEditSSOSettings(); // Enhanced form submission handler const handleFormSubmit = async (formValues: Record) => { - if (!accessToken) { - NotificationsManager.fromBackend("No access token available"); - return; - } + const payload = processSSOSettingsPayload(formValues); - setIsSubmitting(true); - try { - // Save SSO settings using the new API - await updateSSOSettings(accessToken, formValues); - - NotificationsManager.success("SSO settings added successfully"); - - // Reset form and close modal - form.resetFields(); - onSuccess(); - } catch (error: unknown) { - NotificationsManager.fromBackend("Failed to save SSO settings: " + parseErrorMessage(error)); - } finally { - setIsSubmitting(false); - } - }; - - // Helper function to render provider fields - const renderProviderFields = (provider: string) => { - const config = ssoProviderConfigs[provider]; - if (!config) return null; - - return config.fields.map((field) => ( - - {field.name.includes("client") ? : } - - )); + await mutateAsync(payload, { + onSuccess: () => { + NotificationsManager.success("SSO settings added successfully"); + onSuccess(); + }, + onError: (error) => { + NotificationsManager.fromBackend("Failed to save SSO settings: " + parseErrorMessage(error)); + }, + }); }; const handleCancel = () => { @@ -148,91 +39,23 @@ const AddSSOSettingsModal: React.FC = ({ isVisible, on }; return ( - -
- - - - - prevValues.sso_provider !== currentValues.sso_provider} - > - {({ getFieldValue }) => { - const provider = getFieldValue("sso_provider"); - return provider ? renderProviderFields(provider) : null; - }} - - - - - - value?.trim()} - rules={[ - { required: true, message: "Please enter the proxy base url" }, - { - pattern: /^https?:\/\/.+/, - message: "URL must start with http:// or https://", - }, - { - validator: (_, value) => { - // Only check for trailing slash if the URL starts with http:// or https:// - if (value && /^https?:\/\/.+/.test(value) && value.endsWith("/")) { - return Promise.reject("URL must not end with a trailing slash"); - } - return Promise.resolve(); - }, - }, - ]} - > - - - -
- Cancel - - Add SSO - -
-
+ + + + + } + onCancel={handleCancel} + > + ); }; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx new file mode 100644 index 00000000000..a4b36e5190e --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/BaseSSOSettingsForm.tsx @@ -0,0 +1,249 @@ +"use client"; + +import { TextInput } from "@tremor/react"; +import { Checkbox, Form, Input, Select } from "antd"; +import React from "react"; +import { ssoProviderLogoMap, ssoProviderDisplayNames } from "../constants"; + +export interface BaseSSOSettingsFormProps { + form: any; // Replace with proper Form type if available + onFormSubmit: (formValues: Record) => Promise; +} + +// Define the SSO provider configuration type +export interface SSOProviderConfig { + envVarMap: Record; + fields: Array<{ + label: string; + name: string; + placeholder?: string; + }>; +} + +// Define configurations for each SSO provider +export const ssoProviderConfigs: Record = { + google: { + envVarMap: { + google_client_id: "GOOGLE_CLIENT_ID", + google_client_secret: "GOOGLE_CLIENT_SECRET", + }, + fields: [ + { label: "Google Client ID", name: "google_client_id" }, + { label: "Google Client Secret", name: "google_client_secret" }, + ], + }, + microsoft: { + envVarMap: { + microsoft_client_id: "MICROSOFT_CLIENT_ID", + microsoft_client_secret: "MICROSOFT_CLIENT_SECRET", + microsoft_tenant: "MICROSOFT_TENANT", + }, + fields: [ + { label: "Microsoft Client ID", name: "microsoft_client_id" }, + { label: "Microsoft Client Secret", name: "microsoft_client_secret" }, + { label: "Microsoft Tenant", name: "microsoft_tenant" }, + ], + }, + okta: { + envVarMap: { + generic_client_id: "GENERIC_CLIENT_ID", + generic_client_secret: "GENERIC_CLIENT_SECRET", + generic_authorization_endpoint: "GENERIC_AUTHORIZATION_ENDPOINT", + generic_token_endpoint: "GENERIC_TOKEN_ENDPOINT", + generic_userinfo_endpoint: "GENERIC_USERINFO_ENDPOINT", + }, + fields: [ + { label: "Generic Client ID", name: "generic_client_id" }, + { label: "Generic Client Secret", name: "generic_client_secret" }, + { + label: "Authorization Endpoint", + name: "generic_authorization_endpoint", + placeholder: "https://your-domain/authorize", + }, + { label: "Token Endpoint", name: "generic_token_endpoint", placeholder: "https://your-domain/token" }, + { + label: "Userinfo Endpoint", + name: "generic_userinfo_endpoint", + placeholder: "https://your-domain/userinfo", + }, + ], + }, + generic: { + envVarMap: { + generic_client_id: "GENERIC_CLIENT_ID", + generic_client_secret: "GENERIC_CLIENT_SECRET", + generic_authorization_endpoint: "GENERIC_AUTHORIZATION_ENDPOINT", + generic_token_endpoint: "GENERIC_TOKEN_ENDPOINT", + generic_userinfo_endpoint: "GENERIC_USERINFO_ENDPOINT", + }, + fields: [ + { label: "Generic Client ID", name: "generic_client_id" }, + { label: "Generic Client Secret", name: "generic_client_secret" }, + { label: "Authorization Endpoint", name: "generic_authorization_endpoint" }, + { label: "Token Endpoint", name: "generic_token_endpoint" }, + { label: "Userinfo Endpoint", name: "generic_userinfo_endpoint" }, + ], + }, +}; + +// Helper function to render provider fields +export const renderProviderFields = (provider: string) => { + const config = ssoProviderConfigs[provider]; + if (!config) return null; + + return config.fields.map((field) => ( + + {field.name.includes("client") ? : } + + )); +}; + +const BaseSSOSettingsForm: React.FC = ({ form, onFormSubmit }) => { + return ( +
+
+ + + + + prevValues.sso_provider !== currentValues.sso_provider} + > + {({ getFieldValue }) => { + const provider = getFieldValue("sso_provider"); + return provider ? renderProviderFields(provider) : null; + }} + + + + + + value?.trim()} + rules={[ + { required: true, message: "Please enter the proxy base url" }, + { + pattern: /^https?:\/\/.+/, + message: "URL must start with http:// or https://", + }, + { + validator: (_, value) => { + // Only check for trailing slash if the URL starts with http:// or https:// + if (value && /^https?:\/\/.+/.test(value) && value.endsWith("/")) { + return Promise.reject("URL must not end with a trailing slash"); + } + return Promise.resolve(); + }, + }, + ]} + > + + + + prevValues.sso_provider !== currentValues.sso_provider} + > + {({ getFieldValue }) => { + const provider = getFieldValue("sso_provider"); + return provider === "okta" || provider === "generic" ? ( + + + + ) : null; + }} + + + prevValues.use_role_mappings !== currentValues.use_role_mappings} + > + {({ getFieldValue }) => { + const useRoleMappings = getFieldValue("use_role_mappings"); + return useRoleMappings ? ( + + + + ) : null; + }} + + + prevValues.use_role_mappings !== currentValues.use_role_mappings} + > + {({ getFieldValue }) => { + const useRoleMappings = getFieldValue("use_role_mappings"); + return useRoleMappings ? ( + <> + + + + + + + + + + + + + + + + + + + + + ) : null; + }} + +
+
+ ); +}; + +export default BaseSSOSettingsForm; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx new file mode 100644 index 00000000000..297698a7ba0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx @@ -0,0 +1,131 @@ +"use client"; + +import { Button, Form, Modal, Space } from "antd"; +import React, { useEffect } from "react"; +import BaseSSOSettingsForm from "./BaseSSOSettingsForm"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { parseErrorMessage } from "@/components/shared/errorUtils"; +import { processSSOSettingsPayload } from "../utils"; +import { useSSOSettings } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; +import { useEditSSOSettings } from "@/app/(dashboard)/hooks/sso/useEditSSOSettings"; + +interface EditSSOSettingsModalProps { + isVisible: boolean; + onCancel: () => void; + onSuccess: () => void; +} + +const EditSSOSettingsModal: React.FC = ({ isVisible, onCancel, onSuccess }) => { + const [form] = Form.useForm(); + + // Use react-query hooks for SSO settings + const ssoSettings = useSSOSettings(); + const { mutateAsync, isPending } = useEditSSOSettings(); + useEffect(() => { + if (isVisible && ssoSettings.data && ssoSettings.data.values) { + const ssoData = ssoSettings.data; + console.log("Raw SSO data received:", ssoData); // Debug log + console.log("SSO values:", ssoData.values); // Debug log + console.log("user_email from API:", ssoData.values.user_email); // Debug log + + // Determine which SSO provider is configured + let selectedProvider = null; + if (ssoData.values.google_client_id) { + selectedProvider = "google"; + } else if (ssoData.values.microsoft_client_id) { + selectedProvider = "microsoft"; + } else if (ssoData.values.generic_client_id) { + // Check if it looks like Okta based on endpoints + if ( + ssoData.values.generic_authorization_endpoint?.includes("okta") || + ssoData.values.generic_authorization_endpoint?.includes("auth0") + ) { + selectedProvider = "okta"; + } else { + selectedProvider = "generic"; + } + } + + // Extract role mappings if they exist + let roleMappingFields = {}; + if (ssoData.values.role_mappings) { + const roleMappings = ssoData.values.role_mappings; + + // Helper function to join arrays into comma-separated strings + const joinTeams = (teams: string[] | undefined): string => { + if (!teams || teams.length === 0) return ""; + return teams.join(", "); + }; + + roleMappingFields = { + use_role_mappings: true, + group_claim: roleMappings.group_claim, + default_role: roleMappings.default_role || "internal_user", + proxy_admin_teams: joinTeams(roleMappings.roles?.proxy_admin), + admin_viewer_teams: joinTeams(roleMappings.roles?.proxy_admin_viewer), + internal_user_teams: joinTeams(roleMappings.roles?.internal_user), + internal_viewer_teams: joinTeams(roleMappings.roles?.internal_user_viewer), + }; + } + + // Set form values with existing data (excluding UI access control fields) + const formValues = { + sso_provider: selectedProvider, + ...ssoData.values, + ...roleMappingFields, + }; + + console.log("Setting form values:", formValues); // Debug log + + // Clear form first, then set values with a small delay to ensure proper initialization + form.resetFields(); + setTimeout(() => { + form.setFieldsValue(formValues); + console.log("Form values set, current form values:", form.getFieldsValue()); // Debug log + }, 100); + } + }, [isVisible, ssoSettings.data, form]); + + // Enhanced form submission handler + const handleFormSubmit = async (formValues: Record) => { + const payload = processSSOSettingsPayload(formValues); + + await mutateAsync(payload, { + onSuccess: () => { + NotificationsManager.success("SSO settings updated successfully"); + onSuccess(); + }, + onError: (error) => { + NotificationsManager.fromBackend("Failed to save SSO settings: " + parseErrorMessage(error)); + }, + }); + }; + + const handleCancel = () => { + form.resetFields(); + onCancel(); + }; + + return ( + + + + + } + onCancel={handleCancel} + > + + + ); +}; + +export default EditSSOSettingsModal; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.tsx new file mode 100644 index 00000000000..44fef5cc7f8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.tsx @@ -0,0 +1,38 @@ +import { useState } from "react"; +import { Button } from "antd"; +import { Eye, EyeOff } from "lucide-react"; + +export default function RedactableField({ + defaultHidden = true, + value, +}: { + defaultHidden?: boolean; + value: string | null; +}) { + const [isHidden, setIsHidden] = useState(defaultHidden); + + return ( +
+ + {value ? ( + isHidden ? ( + "•".repeat(value.length) + ) : ( + value + ) + ) : ( + Not configured + )} + + {value && ( +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx index 0ece207662c..975a3bc7d78 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx @@ -3,11 +3,14 @@ import { useSSOSettings, type SSOSettingsValues } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { Badge, Button, Card, Descriptions, Space, Typography } from "antd"; -import { Shield, Trash2 } from "lucide-react"; +import { Shield, Trash2, Edit } from "lucide-react"; import { useState } from "react"; import AddSSOSettingsModal from "./Modals/AddSSOSettingsModal"; import DeleteSSOSettingsModal from "./Modals/DeleteSSOSettingsModal"; +import EditSSOSettingsModal from "./Modals/EditSSOSettingsModal"; import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; +import RedactableField from "./RedactableField"; +import { ssoProviderLogoMap, ssoProviderDisplayNames } from "./constants"; const { Title, Text } = Typography; @@ -16,6 +19,7 @@ export default function SSOSettings() { const { accessToken } = useAuthorized(); const [isDeleteModalVisible, setIsDeleteModalVisible] = useState(false); const [isAddModalVisible, setIsAddModalVisible] = useState(false); + const [isEditModalVisible, setIsEditModalVisible] = useState(false); const isSSOConfigured = Boolean(ssoSettings?.values.google_client_id) || Boolean(ssoSettings?.values.microsoft_client_id) || @@ -39,12 +43,6 @@ export default function SSOSettings() { } } - const renderRedactedValue = (value?: string | null) => ( - - {value ? "••••••••••••••••••••••••••••••••" : Not configured} - - ); - const renderEndpointValue = (value?: string | null) => ( {value || Not configured} @@ -67,44 +65,44 @@ export default function SSOSettings() { const providerConfigs = { google: { - providerText: "Google OAuth", + providerText: ssoProviderDisplayNames.google, fields: [ { - label: "Client ID (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.google_client_id), + label: "Client ID", + render: (values: SSOSettingsValues) => , }, { - label: "Client Secret (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.google_client_secret), + label: "Client Secret", + render: (values: SSOSettingsValues) => , }, { label: "Proxy Base URL", render: (values: SSOSettingsValues) => renderSimpleValue(values.proxy_base_url) }, ], }, microsoft: { - providerText: "Microsoft OAuth", + providerText: ssoProviderDisplayNames.microsoft, fields: [ { - label: "Client ID (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.microsoft_client_id), + label: "Client ID", + render: (values: SSOSettingsValues) => , }, { - label: "Client Secret (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.microsoft_client_secret), + label: "Client Secret", + render: (values: SSOSettingsValues) => , }, { label: "Tenant", render: (values: any) => renderSimpleValue(values.microsoft_tenant) }, { label: "Proxy Base URL", render: (values: SSOSettingsValues) => renderSimpleValue(values.proxy_base_url) }, ], }, okta: { - providerText: "Okta/Auth0", + providerText: ssoProviderDisplayNames.okta, fields: [ { - label: "Client ID (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.generic_client_id), + label: "Client ID", + render: (values: SSOSettingsValues) => , }, { - label: "Client Secret (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.generic_client_secret), + label: "Client Secret", + render: (values: SSOSettingsValues) => , }, { label: "Authorization Endpoint", @@ -122,15 +120,15 @@ export default function SSOSettings() { ], }, generic: { - providerText: "Generic OAuth", + providerText: ssoProviderDisplayNames.generic, fields: [ { - label: "Client ID (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.generic_client_id), + label: "Client ID", + render: (values: SSOSettingsValues) => , }, { - label: "Client Secret (Redacted)", - render: (values: SSOSettingsValues) => renderRedactedValue(values.generic_client_secret), + label: "Client Secret", + render: (values: SSOSettingsValues) => , }, { label: "Authorization Endpoint", @@ -160,7 +158,16 @@ export default function SSOSettings() { return ( - +
+ {ssoProviderLogoMap[selectedProvider] && ( + {selectedProvider} + )} + {config.providerText} +
{config.fields.map((field, index) => ( @@ -186,9 +193,14 @@ export default function SSOSettings() {
{isSSOConfigured && ( - + <> + + + )}
@@ -214,7 +226,15 @@ export default function SSOSettings() { setIsAddModalVisible(false); refetch(); }} - accessToken={accessToken} + /> + + setIsEditModalVisible(false)} + onSuccess={() => { + setIsEditModalVisible(false); + refetch(); + }} /> ); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts new file mode 100644 index 00000000000..595a961401a --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/constants.ts @@ -0,0 +1,15 @@ +// SSO Provider logos +export const ssoProviderLogoMap: Record = { + google: "https://artificialanalysis.ai/img/logos/google_small.svg", + microsoft: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", + okta: "https://www.okta.com/sites/default/files/Okta_Logo_BrightBlue_Medium.png", + generic: "", +}; + +// SSO Provider display names (consistent between select dropdown and table) +export const ssoProviderDisplayNames: Record = { + google: "Google SSO", + microsoft: "Microsoft SSO", + okta: "Okta / Auth0 SSO", + generic: "Generic SSO", +}; diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts new file mode 100644 index 00000000000..1c878d7b7b3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.test.ts @@ -0,0 +1,274 @@ +import { processSSOSettingsPayload } from "./utils"; +import { describe, it, expect } from "vitest"; + +describe("processSSOSettingsPayload", () => { + describe("without role mappings", () => { + it("should return all fields except role mapping fields when use_role_mappings is false", () => { + const formValues = { + proxy_admin_teams: "team1, team2", + admin_viewer_teams: "viewer1", + internal_user_teams: "user1", + internal_viewer_teams: "viewer1", + default_role: "proxy_admin", + group_claim: "groups", + use_role_mappings: false, + other_field: "value", + another_field: 123, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result).toEqual({ + other_field: "value", + another_field: 123, + }); + expect(result.role_mappings).toBeUndefined(); + }); + + it("should return all fields except role mapping fields when use_role_mappings is not present", () => { + const formValues = { + proxy_admin_teams: "team1", + admin_viewer_teams: "viewer1", + internal_user_teams: "user1", + internal_viewer_teams: "viewer1", + default_role: "proxy_admin", + group_claim: "groups", + other_field: "value", + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result).toEqual({ + other_field: "value", + }); + expect(result.role_mappings).toBeUndefined(); + }); + }); + + describe("with role mappings enabled", () => { + it("should create role mappings with all team types populated", () => { + const formValues = { + proxy_admin_teams: "admin1, admin2", + admin_viewer_teams: "viewer1, viewer2, viewer3", + internal_user_teams: "user1", + internal_viewer_teams: "internal_viewer1, internal_viewer2", + default_role: "proxy_admin", + group_claim: "groups", + use_role_mappings: true, + other_field: "value", + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.other_field).toBe("value"); + expect(result.role_mappings).toEqual({ + provider: "generic", + group_claim: "groups", + default_role: "proxy_admin", + roles: { + proxy_admin: ["admin1", "admin2"], + proxy_admin_viewer: ["viewer1", "viewer2", "viewer3"], + internal_user: ["user1"], + internal_user_viewer: ["internal_viewer1", "internal_viewer2"], + }, + }); + }); + + it("should handle empty team strings", () => { + const formValues = { + proxy_admin_teams: "", + admin_viewer_teams: "", + internal_user_teams: "", + internal_viewer_teams: "", + default_role: "internal_user", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.roles).toEqual({ + proxy_admin: [], + proxy_admin_viewer: [], + internal_user: [], + internal_user_viewer: [], + }); + }); + + it("should handle undefined team fields", () => { + const formValues = { + default_role: "internal_user_viewer", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.roles).toEqual({ + proxy_admin: [], + proxy_admin_viewer: [], + internal_user: [], + internal_user_viewer: [], + }); + }); + + it("should handle whitespace-only team strings", () => { + const formValues = { + proxy_admin_teams: " ", + admin_viewer_teams: ", , ,", + internal_user_teams: "user1, , user2", + internal_viewer_teams: "viewer1, ,viewer2", + default_role: "proxy_admin_viewer", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.roles).toEqual({ + proxy_admin: [], + proxy_admin_viewer: [], + internal_user: ["user1", "user2"], + internal_user_viewer: ["viewer1", "viewer2"], + }); + }); + + it("should trim whitespace from team names", () => { + const formValues = { + proxy_admin_teams: " admin1 , admin2 ", + admin_viewer_teams: " viewer1 ", + internal_user_teams: " user1 , user2 ", + internal_viewer_teams: "viewer1,viewer2", + default_role: "internal_user", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.roles).toEqual({ + proxy_admin: ["admin1", "admin2"], + proxy_admin_viewer: ["viewer1"], + internal_user: ["user1", "user2"], + internal_user_viewer: ["viewer1", "viewer2"], + }); + }); + + it("should filter out empty strings after trimming", () => { + const formValues = { + proxy_admin_teams: "admin1,,admin2, , admin3", + default_role: "internal_user", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.roles.proxy_admin).toEqual(["admin1", "admin2", "admin3"]); + }); + }); + + describe("default role mapping", () => { + it("should map internal_user_viewer correctly", () => { + const formValues = { + default_role: "internal_user_viewer", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.default_role).toBe("internal_user_viewer"); + }); + + it("should map internal_user correctly", () => { + const formValues = { + default_role: "internal_user", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.default_role).toBe("internal_user"); + }); + + it("should map proxy_admin_viewer correctly", () => { + const formValues = { + default_role: "proxy_admin_viewer", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.default_role).toBe("proxy_admin_viewer"); + }); + + it("should map proxy_admin correctly", () => { + const formValues = { + default_role: "proxy_admin", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.default_role).toBe("proxy_admin"); + }); + + it("should default to internal_user for unknown roles", () => { + const formValues = { + default_role: "unknown_role", + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.default_role).toBe("internal_user"); + }); + + it("should default to internal_user for undefined default_role", () => { + const formValues = { + group_claim: "groups", + use_role_mappings: true, + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result.role_mappings.default_role).toBe("internal_user"); + }); + }); + + describe("edge cases", () => { + it("should handle empty form values", () => { + const result = processSSOSettingsPayload({}); + + expect(result).toEqual({}); + }); + + it("should preserve other fields in the payload", () => { + const formValues = { + use_role_mappings: false, + sso_provider: "google", + client_id: "123", + client_secret: "secret", + redirect_url: "http://example.com", + custom_field: { nested: "value" }, + array_field: [1, 2, 3], + }; + + const result = processSSOSettingsPayload(formValues); + + expect(result).toEqual({ + sso_provider: "google", + client_id: "123", + client_secret: "secret", + redirect_url: "http://example.com", + custom_field: { nested: "value" }, + array_field: [1, 2, 3], + }); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts new file mode 100644 index 00000000000..3533e1226c2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/utils.ts @@ -0,0 +1,54 @@ +/** + * Processes SSO settings form values and transforms them into the payload format expected by the API + * Handles role mappings transformation and field extraction + */ +export const processSSOSettingsPayload = (formValues: Record): Record => { + const { + proxy_admin_teams, + admin_viewer_teams, + internal_user_teams, + internal_viewer_teams, + default_role, + group_claim, + use_role_mappings, + ...rest + } = formValues; + + const payload: any = { + ...rest, + }; + + // Add role mappings if use_role_mappings is checked + if (use_role_mappings) { + // Helper function to split comma-separated string into array + const splitTeams = (teams: string | undefined): string[] => { + if (!teams || teams.trim() === "") return []; + return teams + .split(",") + .map((team) => team.trim()) + .filter((team) => team.length > 0); + }; + + // Map default role display values to backend values + const defaultRoleMapping: Record = { + internal_user_viewer: "internal_user_viewer", + internal_user: "internal_user", + proxy_admin_viewer: "proxy_admin_viewer", + proxy_admin: "proxy_admin", + }; + + payload.role_mappings = { + provider: "generic", + group_claim, + default_role: defaultRoleMapping[default_role] || "internal_user", + roles: { + proxy_admin: splitTeams(proxy_admin_teams), + proxy_admin_viewer: splitTeams(admin_viewer_teams), + internal_user: splitTeams(internal_user_teams), + internal_user_viewer: splitTeams(internal_viewer_teams), + }, + }; + } + + return payload; +}; From 5c7523b11e3bf1786addd39ed71246f2c8c08b50 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 12:43:13 -0800 Subject: [PATCH 224/388] Fixing tests --- .../Modals/AddSSOSettingsModal.test.tsx | 23 +++- .../SSOSettings/RedactableField.test.tsx | 108 ++++++++++++++++++ .../AdminSettings/SSOSettings/SSOSettings.tsx | 8 +- 3 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.test.tsx diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx index aae28191031..e5a5af6cba6 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/AddSSOSettingsModal.test.tsx @@ -1,5 +1,6 @@ -import { render, screen } from "@testing-library/react"; +import { screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; +import { renderWithProviders } from "../../../../../../tests/test-utils"; import AddSSOSettingsModal from "./AddSSOSettingsModal"; // Mock networking functions @@ -12,12 +13,30 @@ vi.mock("@/components/shared/errorUtils", () => ({ parseErrorMessage: vi.fn((error) => error?.message || "Unknown error"), })); +// Mock the useAuthorized hook +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ + default: () => ({ + accessToken: "test-access-token", + userId: "test-user-id", + userEmail: "test@example.com", + userRole: "admin", + }), +})); + +// Mock NotificationsManager +vi.mock("@/components/molecules/notifications_manager", () => ({ + default: { + success: vi.fn(), + fromBackend: vi.fn(), + }, +})); + describe("AddSSOSettingsModal", () => { it("should render", () => { const onCancel = vi.fn(); const onSuccess = vi.fn(); - render(); + renderWithProviders(); expect(screen.getByText("SSO Provider")).toBeInTheDocument(); expect(screen.getByText("Cancel")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.test.tsx new file mode 100644 index 00000000000..a047d7aea4f --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/RedactableField.test.tsx @@ -0,0 +1,108 @@ +import { render, screen, fireEvent } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import RedactableField from "./RedactableField"; + +describe("RedactableField", () => { + describe("when value is null", () => { + it("should display 'Not configured' text", () => { + render(); + + expect(screen.getByText("Not configured")).toBeInTheDocument(); + }); + + it("should not display toggle button", () => { + render(); + + // There should be no button elements + const buttons = screen.queryAllByRole("button"); + expect(buttons).toHaveLength(0); + }); + }); + + describe("when value is provided", () => { + const testValue = "secret-password"; + + it("should be hidden by default and show redacted dots", () => { + render(); + + // Should show dots equal to the length of the value + expect(screen.getByText("•".repeat(testValue.length))).toBeInTheDocument(); + expect(screen.queryByText(testValue)).not.toBeInTheDocument(); + }); + + it("should show actual value when defaultHidden is false", () => { + render(); + + expect(screen.getByText(testValue)).toBeInTheDocument(); + expect(screen.queryByText("•".repeat(testValue.length))).not.toBeInTheDocument(); + }); + + it("should display toggle button with eye icon when hidden", () => { + render(); + + const button = screen.getByRole("button"); + expect(button).toBeInTheDocument(); + + // Check that the Eye icon is rendered (we can check by title or by the presence of the icon) + // The button should contain the Eye icon when hidden + const eyeIcon = button.querySelector("svg"); + expect(eyeIcon).toBeInTheDocument(); + }); + + it("should display toggle button with eye-off icon when shown", () => { + render(); + + const button = screen.getByRole("button"); + expect(button).toBeInTheDocument(); + + // The button should contain the EyeOff icon when shown + const eyeOffIcon = button.querySelector("svg"); + expect(eyeOffIcon).toBeInTheDocument(); + }); + + it("should toggle visibility when button is clicked", () => { + render(); + + // Initially hidden + expect(screen.getByText("•".repeat(testValue.length))).toBeInTheDocument(); + expect(screen.queryByText(testValue)).not.toBeInTheDocument(); + + // Click to show + const button = screen.getByRole("button"); + fireEvent.click(button); + + // Should now show the actual value + expect(screen.getByText(testValue)).toBeInTheDocument(); + expect(screen.queryByText("•".repeat(testValue.length))).not.toBeInTheDocument(); + + // Click again to hide + fireEvent.click(button); + + // Should be hidden again + expect(screen.getByText("•".repeat(testValue.length))).toBeInTheDocument(); + expect(screen.queryByText(testValue)).not.toBeInTheDocument(); + }); + + it("should handle empty string value", () => { + render(); + + // Empty string should show "Not configured" since value is falsy + expect(screen.getByText("Not configured")).toBeInTheDocument(); + + // No toggle button for empty string + const buttons = screen.queryAllByRole("button"); + expect(buttons).toHaveLength(0); + }); + + it("should handle different value lengths correctly", () => { + const shortValue = "hi"; + const longValue = "this-is-a-very-long-secret-value"; + + const { rerender } = render(); + expect(screen.getByText("••")).toBeInTheDocument(); + + rerender(); + expect(screen.getByText("•".repeat(longValue.length))).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx index 975a3bc7d78..d339f6d0e36 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx @@ -2,15 +2,15 @@ import { useSSOSettings, type SSOSettingsValues } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { Badge, Button, Card, Descriptions, Space, Typography } from "antd"; -import { Shield, Trash2, Edit } from "lucide-react"; +import { Button, Card, Descriptions, Space, Typography } from "antd"; +import { Edit, Shield, Trash2 } from "lucide-react"; import { useState } from "react"; import AddSSOSettingsModal from "./Modals/AddSSOSettingsModal"; import DeleteSSOSettingsModal from "./Modals/DeleteSSOSettingsModal"; import EditSSOSettingsModal from "./Modals/EditSSOSettingsModal"; -import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; import RedactableField from "./RedactableField"; -import { ssoProviderLogoMap, ssoProviderDisplayNames } from "./constants"; +import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; +import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; const { Title, Text } = Typography; From 2cbcaf2abf463a137e9a7f164c305da15e6c9413 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 13:34:17 -0800 Subject: [PATCH 225/388] refactor(utils): lazy load heavy imports to improve import time and memory usage (#18610) * refactor(utils): lazy load heavy imports to improve import time - Move BaseVectorStore, CredentialAccessor, and exception_mapping_utils imports to lazy loading via __getattr__ - Add _get_utils_globals() helper function following pattern from _lazy_imports.py - Refactor __getattr__ to use consistent caching pattern matching __init__.py - Update load_credentials_from_list to use lazy-loaded CredentialAccessor This reduces import time and memory usage by only loading these modules when they're actually accessed, not during module import. * refactor(utils): lazy load additional heavy imports to improve import time - Move get_llm_provider, _is_non_openai_azure_model to lazy loading - Move get_supported_openai_params to lazy loading - Move convert_dict_to_response functions (LiteLLMResponseObjectHandler, convert_to_model_response_object, etc.) to lazy loading - Move get_api_base and ResponseMetadata to lazy loading - Move _parse_content_for_reasoning to lazy loading - Update all internal usages to access via getattr(sys.modules[__name__], ...) This reduces import time and memory usage by only loading these modules when they're actually accessed, not during module import. * fix(utils): suppress PLR0915 linter warning for __getattr__ function The __getattr__ function intentionally has many statements to handle multiple lazy-loaded imports. Add noqa comment to suppress the warning. * fix(utils): add type stubs for lazy-loaded functions in TYPE_CHECKING block Add type imports and declarations in TYPE_CHECKING block to help mypy understand the types of lazy-loaded functions accessed via __getattr__. This follows the same pattern used in __init__.py for lazy-loaded items. --- litellm/utils.py | 249 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 209 insertions(+), 40 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 3dbeeb970a4..e5eb57b0712 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -71,9 +71,6 @@ from litellm.constants import ( OPENAI_EMBEDDING_PARAMS, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -from litellm.integrations.vector_store_integrations.base_vector_store import ( - BaseVectorStore, -) # Import cached imports utilities from litellm.litellm_core_utils.cached_imports import ( @@ -86,48 +83,21 @@ from litellm.litellm_core_utils.core_helpers import ( map_finish_reason, process_response_headers, ) -from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.dot_notation_indexing import ( delete_nested_value, is_nested_path, ) -from litellm.litellm_core_utils.exception_mapping_utils import ( - _get_response_headers, - exception_type, - get_error_message, -) from litellm.litellm_core_utils.get_litellm_params import ( _get_base_model_from_litellm_call_metadata, get_litellm_params, ) -from litellm.litellm_core_utils.get_llm_provider_logic import ( - _is_non_openai_azure_model, - get_llm_provider, -) -from litellm.litellm_core_utils.get_supported_openai_params import ( - get_supported_openai_params, -) from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe -from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( - LiteLLMResponseObjectHandler, - _handle_invalid_parallel_tool_calls, - convert_to_model_response_object, - convert_to_streaming_response, - convert_to_streaming_response_async, -) -from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( get_formatted_prompt, ) from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - ResponseMetadata, -) -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - _parse_content_for_reasoning, -) from litellm.litellm_core_utils.redact_messages import ( LiteLLMLoggingObject, redact_message_input_output_from_logging, @@ -346,6 +316,30 @@ if TYPE_CHECKING: from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.proxy._types import AllowedModelRegion + # Type stubs for lazy-loaded functions to help mypy understand their types + # These imports allow mypy to understand the types when these are accessed via __getattr__ + from litellm.litellm_core_utils.exception_mapping_utils import exception_type + from litellm.litellm_core_utils.get_llm_provider_logic import ( + _is_non_openai_azure_model, + get_llm_provider, + ) + from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params, + ) + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + LiteLLMResponseObjectHandler, + _handle_invalid_parallel_tool_calls, + convert_to_model_response_object, + convert_to_streaming_response, + convert_to_streaming_response_async, + ) + from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + ResponseMetadata, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning, + ) from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig @@ -618,10 +612,22 @@ def get_applied_guardrails(kwargs: Dict[str, Any]) -> List[str]: return applied_guardrails +def _get_utils_globals() -> dict: + """ + Get the globals dictionary of the utils module. + + This is where we cache imported attributes so we don't import them twice. + """ + return sys.modules[__name__].__dict__ + + def load_credentials_from_list(kwargs: dict): """ Updates kwargs with the credentials if credential_name in kwarg """ + # Access CredentialAccessor via module to trigger lazy loading if needed + CredentialAccessor = getattr(sys.modules[__name__], 'CredentialAccessor') + credential_name = kwargs.get("litellm_credential_name") if credential_name and litellm.credential_list: credential_accessor = CredentialAccessor.get_credential_values(credential_name) @@ -2259,6 +2265,7 @@ def supports_response_schema( """ ## GET LLM PROVIDER ## try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') model, custom_llm_provider, _, _ = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider ) @@ -2956,6 +2963,9 @@ def get_optional_params_embeddings( # noqa: PLR0915 additional_drop_params: Optional[List[str]] = None, **kwargs, ): + # Lazy load get_supported_openai_params + get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params') + # retrieve all parameters passed to the function passed_params = locals() custom_llm_provider = passed_params.pop("custom_llm_provider", None) @@ -3758,6 +3768,7 @@ def get_optional_params( # noqa: PLR0915 message=f"{custom_llm_provider} does not support parameters: {list(unsupported_params.keys())}, for model={model}. To drop these, set `litellm.drop_params=True` or for proxy:\n\n`litellm_settings:\n drop_params: true`\n. \n If you want to use these params dynamically send allowed_openai_params={list(unsupported_params.keys())} in your request.", ) + get_supported_openai_params = getattr(sys.modules[__name__], 'get_supported_openai_params') supported_params = get_supported_openai_params( model=model, custom_llm_provider=custom_llm_provider ) @@ -4895,6 +4906,7 @@ def get_max_tokens(model: str) -> Optional[int]: return litellm.model_cost[model]["max_output_tokens"] elif "max_tokens" in litellm.model_cost[model]: return litellm.model_cost[model]["max_tokens"] + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') model, custom_llm_provider, _, _ = get_llm_provider(model=model) if custom_llm_provider == "huggingface": max_tokens = _get_max_position_embeddings(model_name=model) @@ -5015,6 +5027,7 @@ def _get_potential_model_names( if custom_llm_provider is None: # Get custom_llm_provider try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') split_model, custom_llm_provider, _, _ = get_llm_provider(model=model) except Exception: split_model = model @@ -5737,6 +5750,7 @@ def validate_environment( # noqa: PLR0915 } ## EXTRACT LLM PROVIDER - if model name provided try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') _, custom_llm_provider, _, _ = get_llm_provider(model=model) except Exception: custom_llm_provider = None @@ -6299,6 +6313,7 @@ def register_prompt_template( complete_model = model potential_models = [complete_model] try: + get_llm_provider = getattr(sys.modules[__name__], 'get_llm_provider') model = get_llm_provider(model=model)[0] potential_models.append(model) except Exception: @@ -6384,6 +6399,7 @@ class TextCompletionStreamWrapper: except StopIteration: raise StopIteration except Exception as e: + exception_type = getattr(sys.modules[__name__], 'exception_type') raise exception_type( model=self.model, custom_llm_provider=self.custom_llm_provider or "", @@ -8705,16 +8721,169 @@ def should_run_mock_completion( return False -# Re-export encoding from main.py for backward compatibility -# This allows tests to import: from litellm.utils import encoding -# We use a lazy import to avoid loading main.py at utils.py import time -def __getattr__(name: str) -> Any: +def __getattr__(name: str) -> Any: # noqa: PLR0915 """Lazy import handler for utils module""" + _globals = _get_utils_globals() + + # Lazy load encoding from main.py to avoid heavy tiktoken import if name == "encoding": - # Cache it in the module's __dict__ for subsequent accesses - import sys - - from litellm.main import encoding as _encoding - sys.modules[__name__].__dict__["encoding"] = _encoding - return _encoding + # Check if already cached + if "encoding" not in _globals: + from litellm.main import encoding as _encoding + _globals["encoding"] = _encoding + return _globals["encoding"] + + # Lazy load BaseVectorStore to avoid loading it at module import time + if name == "BaseVectorStore": + # Check if already cached + if "BaseVectorStore" not in _globals: + from litellm.integrations.vector_store_integrations.base_vector_store import ( + BaseVectorStore as _BaseVectorStore, + ) + _globals["BaseVectorStore"] = _BaseVectorStore + return _globals["BaseVectorStore"] + + # Lazy load CredentialAccessor to avoid loading it at module import time + if name == "CredentialAccessor": + # Check if already cached + if "CredentialAccessor" not in _globals: + from litellm.litellm_core_utils.credential_accessor import ( + CredentialAccessor as _CredentialAccessor, + ) + _globals["CredentialAccessor"] = _CredentialAccessor + return _globals["CredentialAccessor"] + + # Lazy load exception_mapping_utils functions to avoid loading at module import time + if name == "exception_type": + # Check if already cached + if "exception_type" not in _globals: + from litellm.litellm_core_utils.exception_mapping_utils import ( + exception_type as _exception_type, + ) + _globals["exception_type"] = _exception_type + return _globals["exception_type"] + + if name == "get_error_message": + # Check if already cached + if "get_error_message" not in _globals: + from litellm.litellm_core_utils.exception_mapping_utils import ( + get_error_message as _get_error_message, + ) + _globals["get_error_message"] = _get_error_message + return _globals["get_error_message"] + + if name == "_get_response_headers": + # Check if already cached + if "_get_response_headers" not in _globals: + from litellm.litellm_core_utils.exception_mapping_utils import ( + _get_response_headers as __get_response_headers, + ) + _globals["_get_response_headers"] = __get_response_headers + return _globals["_get_response_headers"] + + # Lazy load get_llm_provider_logic functions to avoid loading at module import time + if name == "get_llm_provider": + # Check if already cached + if "get_llm_provider" not in _globals: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider as _get_llm_provider, + ) + _globals["get_llm_provider"] = _get_llm_provider + return _globals["get_llm_provider"] + + if name == "_is_non_openai_azure_model": + # Check if already cached + if "_is_non_openai_azure_model" not in _globals: + from litellm.litellm_core_utils.get_llm_provider_logic import ( + _is_non_openai_azure_model as __is_non_openai_azure_model, + ) + _globals["_is_non_openai_azure_model"] = __is_non_openai_azure_model + return _globals["_is_non_openai_azure_model"] + + # Lazy load get_supported_openai_params to avoid loading at module import time + if name == "get_supported_openai_params": + # Check if already cached + if "get_supported_openai_params" not in _globals: + from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params as _get_supported_openai_params, + ) + _globals["get_supported_openai_params"] = _get_supported_openai_params + return _globals["get_supported_openai_params"] + + # Lazy load convert_dict_to_response functions to avoid loading at module import time + if name == "LiteLLMResponseObjectHandler": + # Check if already cached + if "LiteLLMResponseObjectHandler" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + LiteLLMResponseObjectHandler as _LiteLLMResponseObjectHandler, + ) + _globals["LiteLLMResponseObjectHandler"] = _LiteLLMResponseObjectHandler + return _globals["LiteLLMResponseObjectHandler"] + + if name == "_handle_invalid_parallel_tool_calls": + # Check if already cached + if "_handle_invalid_parallel_tool_calls" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + _handle_invalid_parallel_tool_calls as __handle_invalid_parallel_tool_calls, + ) + _globals["_handle_invalid_parallel_tool_calls"] = __handle_invalid_parallel_tool_calls + return _globals["_handle_invalid_parallel_tool_calls"] + + if name == "convert_to_model_response_object": + # Check if already cached + if "convert_to_model_response_object" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_model_response_object as _convert_to_model_response_object, + ) + _globals["convert_to_model_response_object"] = _convert_to_model_response_object + return _globals["convert_to_model_response_object"] + + if name == "convert_to_streaming_response": + # Check if already cached + if "convert_to_streaming_response" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response as _convert_to_streaming_response, + ) + _globals["convert_to_streaming_response"] = _convert_to_streaming_response + return _globals["convert_to_streaming_response"] + + if name == "convert_to_streaming_response_async": + # Check if already cached + if "convert_to_streaming_response_async" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( + convert_to_streaming_response_async as _convert_to_streaming_response_async, + ) + _globals["convert_to_streaming_response_async"] = _convert_to_streaming_response_async + return _globals["convert_to_streaming_response_async"] + + # Lazy load get_api_base to avoid loading at module import time + if name == "get_api_base": + # Check if already cached + if "get_api_base" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.get_api_base import ( + get_api_base as _get_api_base, + ) + _globals["get_api_base"] = _get_api_base + return _globals["get_api_base"] + + # Lazy load ResponseMetadata to avoid loading at module import time + if name == "ResponseMetadata": + # Check if already cached + if "ResponseMetadata" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + ResponseMetadata as _ResponseMetadata, + ) + _globals["ResponseMetadata"] = _ResponseMetadata + return _globals["ResponseMetadata"] + + # Lazy load _parse_content_for_reasoning to avoid loading at module import time + if name == "_parse_content_for_reasoning": + # Check if already cached + if "_parse_content_for_reasoning" not in _globals: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + _parse_content_for_reasoning as __parse_content_for_reasoning, + ) + _globals["_parse_content_for_reasoning"] = __parse_content_for_reasoning + return _globals["_parse_content_for_reasoning"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From 1c5c303e986bb5c11756ab411713badd6c1a1362 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 14:07:57 -0800 Subject: [PATCH 226/388] refactor(utils): implement lazy loading for provider configs, model info classes, streaming handlers, and redact utilities (#18611) * refactor(utils): lazy load redact_messages imports to improve import time - Move LiteLLMLoggingObject and redact_message_input_output_from_logging to lazy loading via __getattr__ - Add type stubs in TYPE_CHECKING block for mypy type checking - These are only used in type annotations (with from __future__ import annotations), so lazy loading works correctly This reduces import time by deferring the redact_messages module import until these are actually accessed. * refactor(utils): lazy load CustomStreamWrapper to improve import time - Move CustomStreamWrapper from streaming_handler to lazy loading via __getattr__ - Add type stub in TYPE_CHECKING block for mypy type checking - CustomStreamWrapper is not used internally in utils.py, only exported for other modules This reduces import time by deferring the streaming_handler module import until CustomStreamWrapper is actually accessed. * refactor(utils): lazy load BaseGoogleGenAIGenerateContentConfig to improve import time - Move BaseGoogleGenAIGenerateContentConfig from google_genai.transformation to lazy loading via __getattr__ - Add type stub in TYPE_CHECKING block for mypy type checking - BaseGoogleGenAIGenerateContentConfig is only used in type annotations (with from __future__ import annotations), so lazy loading works correctly This reduces import time by deferring the google_genai.transformation module import until BaseGoogleGenAIGenerateContentConfig is actually accessed. * refactor(utils): lazy load BaseOCRConfig, BaseSearchConfig, and BaseTextToSpeechConfig - Move BaseOCRConfig, BaseSearchConfig, and BaseTextToSpeechConfig to lazy loading via __getattr__ - Add type stubs in TYPE_CHECKING block for mypy type checking - These config classes are only used in quoted type annotations (forward references), so lazy loading works correctly This reduces import time by deferring the transformation module imports until these config classes are actually accessed. * refactor(utils): lazy load BedrockModelInfo, CohereModelInfo, and MistralOCRConfig - Move BedrockModelInfo, CohereModelInfo, and MistralOCRConfig to lazy loading via __getattr__ - Add type stubs in TYPE_CHECKING block for mypy type checking - Update internal usages to use getattr pattern for accessing lazy-loaded classes - These provider-specific model info classes are only used in specific code paths, so lazy loading reduces initial import time This reduces import time by deferring the bedrock, cohere, and mistral module imports until these classes are actually accessed. * fix(utils): remove duplicate MistralOCRConfig import in TYPE_CHECKING block --- litellm/utils.py | 130 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 116 insertions(+), 14 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index e5eb57b0712..d373b5102ef 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -98,22 +98,8 @@ from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) -from litellm.litellm_core_utils.redact_messages import ( - LiteLLMLoggingObject, - redact_message_input_output_from_logging, -) from litellm.litellm_core_utils.rules import Rules -from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.llms.base_llm.google_genai.transformation import ( - BaseGoogleGenAIGenerateContentConfig, -) -from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig -from litellm.llms.base_llm.search.transformation import BaseSearchConfig -from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig -from litellm.llms.bedrock.common_utils import BedrockModelInfo -from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.llms.mistral.ocr.transformation import MistralOCRConfig from litellm.router_utils.get_retry_from_policy import ( get_num_retries_from_retry_policy, reset_retry_policy, @@ -340,6 +326,20 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.prompt_templates.common_utils import ( _parse_content_for_reasoning, ) + from litellm.litellm_core_utils.redact_messages import ( + LiteLLMLoggingObject, + redact_message_input_output_from_logging, + ) + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.llms.base_llm.google_genai.transformation import ( + BaseGoogleGenAIGenerateContentConfig, + ) + from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig + from litellm.llms.base_llm.search.transformation import BaseSearchConfig + from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig + from litellm.llms.bedrock.common_utils import BedrockModelInfo + from litellm.llms.cohere.common_utils import CohereModelInfo + from litellm.llms.mistral.ocr.transformation import MistralOCRConfig from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig @@ -4023,6 +4023,7 @@ def get_optional_params( # noqa: PLR0915 ), ) elif custom_llm_provider == "bedrock": + BedrockModelInfo = getattr(sys.modules[__name__], 'BedrockModelInfo') bedrock_route = BedrockModelInfo.get_bedrock_route(model) bedrock_base_model = BedrockModelInfo.get_base_model(model) if bedrock_route == "converse" or bedrock_route == "converse_like": @@ -7440,6 +7441,7 @@ class ProviderConfigManager: litellm.LlmProviders.COHERE_CHAT == provider or litellm.LlmProviders.COHERE == provider ): + CohereModelInfo = getattr(sys.modules[__name__], 'CohereModelInfo') route = CohereModelInfo.get_cohere_route(model) if route == "v2": return litellm.CohereV2ChatConfig() @@ -8290,6 +8292,7 @@ class ProviderConfigManager: return get_vertex_ai_ocr_config(model=model) + MistralOCRConfig = getattr(sys.modules[__name__], 'MistralOCRConfig') PROVIDER_TO_CONFIG_MAP = { litellm.LlmProviders.MISTRAL: MistralOCRConfig, } @@ -8886,4 +8889,103 @@ def __getattr__(name: str) -> Any: # noqa: PLR0915 _globals["_parse_content_for_reasoning"] = __parse_content_for_reasoning return _globals["_parse_content_for_reasoning"] + # Lazy load redact_messages to avoid loading at module import time + if name == "LiteLLMLoggingObject": + # Check if already cached + if "LiteLLMLoggingObject" not in _globals: + from litellm.litellm_core_utils.redact_messages import ( + LiteLLMLoggingObject as _LiteLLMLoggingObject, + ) + _globals["LiteLLMLoggingObject"] = _LiteLLMLoggingObject + return _globals["LiteLLMLoggingObject"] + + if name == "redact_message_input_output_from_logging": + # Check if already cached + if "redact_message_input_output_from_logging" not in _globals: + from litellm.litellm_core_utils.redact_messages import ( + redact_message_input_output_from_logging as _redact_message_input_output_from_logging, + ) + _globals["redact_message_input_output_from_logging"] = _redact_message_input_output_from_logging + return _globals["redact_message_input_output_from_logging"] + + # Lazy load CustomStreamWrapper to avoid loading at module import time + if name == "CustomStreamWrapper": + # Check if already cached + if "CustomStreamWrapper" not in _globals: + from litellm.litellm_core_utils.streaming_handler import ( + CustomStreamWrapper as _CustomStreamWrapper, + ) + _globals["CustomStreamWrapper"] = _CustomStreamWrapper + return _globals["CustomStreamWrapper"] + + # Lazy load BaseGoogleGenAIGenerateContentConfig to avoid loading at module import time + if name == "BaseGoogleGenAIGenerateContentConfig": + # Check if already cached + if "BaseGoogleGenAIGenerateContentConfig" not in _globals: + from litellm.llms.base_llm.google_genai.transformation import ( + BaseGoogleGenAIGenerateContentConfig as _BaseGoogleGenAIGenerateContentConfig, + ) + _globals["BaseGoogleGenAIGenerateContentConfig"] = _BaseGoogleGenAIGenerateContentConfig + return _globals["BaseGoogleGenAIGenerateContentConfig"] + + # Lazy load BaseOCRConfig to avoid loading at module import time + if name == "BaseOCRConfig": + # Check if already cached + if "BaseOCRConfig" not in _globals: + from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig as _BaseOCRConfig, + ) + _globals["BaseOCRConfig"] = _BaseOCRConfig + return _globals["BaseOCRConfig"] + + # Lazy load BaseSearchConfig to avoid loading at module import time + if name == "BaseSearchConfig": + # Check if already cached + if "BaseSearchConfig" not in _globals: + from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig as _BaseSearchConfig, + ) + _globals["BaseSearchConfig"] = _BaseSearchConfig + return _globals["BaseSearchConfig"] + + # Lazy load BaseTextToSpeechConfig to avoid loading at module import time + if name == "BaseTextToSpeechConfig": + # Check if already cached + if "BaseTextToSpeechConfig" not in _globals: + from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig as _BaseTextToSpeechConfig, + ) + _globals["BaseTextToSpeechConfig"] = _BaseTextToSpeechConfig + return _globals["BaseTextToSpeechConfig"] + + # Lazy load BedrockModelInfo to avoid loading at module import time + if name == "BedrockModelInfo": + # Check if already cached + if "BedrockModelInfo" not in _globals: + from litellm.llms.bedrock.common_utils import ( + BedrockModelInfo as _BedrockModelInfo, + ) + _globals["BedrockModelInfo"] = _BedrockModelInfo + return _globals["BedrockModelInfo"] + + # Lazy load CohereModelInfo to avoid loading at module import time + if name == "CohereModelInfo": + # Check if already cached + if "CohereModelInfo" not in _globals: + from litellm.llms.cohere.common_utils import ( + CohereModelInfo as _CohereModelInfo, + ) + _globals["CohereModelInfo"] = _CohereModelInfo + return _globals["CohereModelInfo"] + + # Lazy load MistralOCRConfig to avoid loading at module import time + if name == "MistralOCRConfig": + # Check if already cached + if "MistralOCRConfig" not in _globals: + from litellm.llms.mistral.ocr.transformation import ( + MistralOCRConfig as _MistralOCRConfig, + ) + _globals["MistralOCRConfig"] = _MistralOCRConfig + return _globals["MistralOCRConfig"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From a6c3fb1fb598422e251fa5bd13e2aa4d16c7d3a6 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 15:10:52 -0800 Subject: [PATCH 227/388] E2E test see models for specific provider --- ui/litellm-dashboard/e2e_tests/constants.ts | 1 + .../tests/modelsPage/addModel.spec.ts | 23 +++++++++++++++++++ .../tests/navigation/sidebar.spec.ts | 3 ++- 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 ui/litellm-dashboard/e2e_tests/constants.ts create mode 100644 ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts diff --git a/ui/litellm-dashboard/e2e_tests/constants.ts b/ui/litellm-dashboard/e2e_tests/constants.ts new file mode 100644 index 00000000000..b07bd68fcf1 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/constants.ts @@ -0,0 +1 @@ +export const ADMIN_STORAGE_PATH = "admin.storageState.json"; diff --git a/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts new file mode 100644 index 00000000000..5fa11a98ef6 --- /dev/null +++ b/ui/litellm-dashboard/e2e_tests/tests/modelsPage/addModel.spec.ts @@ -0,0 +1,23 @@ +import { test, expect } from "@playwright/test"; +import { ADMIN_STORAGE_PATH } from "../../constants"; + +test.describe("Add Model", () => { + test.use({ storageState: ADMIN_STORAGE_PATH }); + + test("Able to see all models for a specific provider in the model dropdown", async ({ page }) => { + await page.goto("http://localhost:4000/ui"); + + await page.getByText("Models + Endpoints").click(); + await page.getByRole("tab", { name: "Add Model" }).click(); + + const providerInputDropdown = page.getByRole("combobox", { name: /Provider/i }); + await providerInputDropdown.fill("Anthropic"); + await page.waitForTimeout(1000); + await providerInputDropdown.press("Enter"); + await page.waitForTimeout(1000); + + const providerModelsDropdown = page.locator(".ant-select-selection-overflow").first(); + await providerModelsDropdown.click(); + await expect(page.getByTitle("claude-haiku-4-5", { exact: true })).toBeVisible(); + }); +}); diff --git a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts index dafb03a7cbd..6801f891e87 100644 --- a/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts +++ b/ui/litellm-dashboard/e2e_tests/tests/navigation/sidebar.spec.ts @@ -1,5 +1,6 @@ import test, { expect } from "@playwright/test"; import { Role } from "../../fixtures/roles"; +import { ADMIN_STORAGE_PATH } from "../../constants"; const sidebarButtons = { [Role.ProxyAdmin]: [ @@ -16,7 +17,7 @@ const sidebarButtons = { ], }; -const roles = [{ role: Role.ProxyAdmin, storage: "admin.storageState.json" }]; +const roles = [{ role: Role.ProxyAdmin, storage: ADMIN_STORAGE_PATH }]; for (const { role, storage } of roles) { test.describe(`${role} sidebar`, () => { From dd1ccec7348b73f9a3d7d1ec8ac5fd29651de759 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 15:50:53 -0800 Subject: [PATCH 228/388] refactor(utils): lazy load 15 additional imports to improve import time (#18613) * refactor(utils): lazy load 15 additional imports to improve import time - Move Rules, AsyncHTTPHandler, HTTPHandler to lazy loading via __getattr__ - Move get_num_retries_from_retry_policy, reset_retry_policy to lazy loading - Move get_secret to lazy loading - Move cached_imports functions (get_coroutine_checker, get_litellm_logging_class, get_set_callbacks) to lazy loading - Move core_helpers functions (get_litellm_metadata_from_kwargs, map_finish_reason, process_response_headers) to lazy loading - Move dot_notation_indexing functions (delete_nested_value, is_nested_path) to lazy loading - Move get_litellm_params functions to lazy loading - Move _ensure_extra_body_is_safe, get_formatted_prompt, get_response_headers, update_response_metadata to lazy loading - Move executor to lazy loading - Move BaseAnthropicMessagesConfig, BaseAudioTranscriptionConfig to lazy loading - Add type stubs in TYPE_CHECKING block for mypy type checking - These functions/classes are exported for other modules but not used internally in utils.py, so lazy loading is safe and improves startup performance * fix(utils): use getattr for Rules and get_coroutine_checker in client decorator - Update Rules() instantiation in client decorator to use getattr for lazy loading - Update Rules.has_pre_call_rules() usage in function_setup to use getattr - Update get_coroutine_checker() usage in client decorator to use getattr - Fixes NameError: name 'Rules' is not defined error that occurs when Rules is lazy-loaded * fix(utils): use getattr for get_litellm_logging_class in function_setup - Update get_litellm_logging_class() usage in function_setup to use getattr for lazy loading - Fixes NameError: name 'get_litellm_logging_class' is not defined error that occurs when get_litellm_logging_class is lazy-loaded * fix(utils): use getattr for get_set_callbacks in function_setup - Update get_set_callbacks() usage in function_setup to use getattr for lazy loading - Fixes NameError: name 'get_set_callbacks' is not defined error that occurs when get_set_callbacks is lazy-loaded * fix(utils): use getattr for all lazy-loaded imports in utils.py - Update update_response_metadata (4 occurrences) to use getattr - Update executor.submit (1 occurrence) to use getattr - Update get_num_retries_from_retry_policy (2 occurrences) to use getattr - Update reset_retry_policy (2 occurrences) to use getattr - Update is_nested_path and delete_nested_value (1 occurrence each) to use getattr - Update _ensure_extra_body_is_safe (1 occurrence) to use getattr Fixes NameError errors that occur when these functions/classes are lazy-loaded but used directly in utils.py * fix(utils): use getattr for _get_base_model_from_litellm_call_metadata in _get_base_model_from_metadata - Update _get_base_model_from_litellm_call_metadata usage to use getattr for lazy loading - Fixes NameError: name '_get_base_model_from_litellm_call_metadata' is not defined * fix(utils): use getattr for second _get_base_model_from_litellm_call_metadata usage - Fix the second occurrence of _get_base_model_from_litellm_call_metadata on line 7052 - Both occurrences in _get_base_model_from_metadata now use getattr for lazy loading * fix(utils): fix indentation in _get_base_model_from_metadata function * fix(utils): use getattr for get_litellm_metadata_from_kwargs in _get_litellm_params - Update get_litellm_metadata_from_kwargs usage to use getattr for lazy loading - Fixes NameError: name 'get_litellm_metadata_from_kwargs' is not defined * fix(utils): fix syntax error in get_litellm_metadata_from_kwargs fix - Move getattr call before cast statement to fix syntax error --- litellm/utils.py | 330 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 284 insertions(+), 46 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index d373b5102ef..6f4652c8278 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -72,39 +72,7 @@ from litellm.constants import ( TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -# Import cached imports utilities -from litellm.litellm_core_utils.cached_imports import ( - get_coroutine_checker, - get_litellm_logging_class, - get_set_callbacks, -) -from litellm.litellm_core_utils.core_helpers import ( - get_litellm_metadata_from_kwargs, - map_finish_reason, - process_response_headers, -) -from litellm.litellm_core_utils.dot_notation_indexing import ( - delete_nested_value, - is_nested_path, -) -from litellm.litellm_core_utils.get_litellm_params import ( - _get_base_model_from_litellm_call_metadata, - get_litellm_params, -) -from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe -from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( - get_formatted_prompt, -) -from litellm.litellm_core_utils.llm_response_utils.get_headers import ( - get_response_headers, -) -from litellm.litellm_core_utils.rules import Rules -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.router_utils.get_retry_from_policy import ( - get_num_retries_from_retry_policy, - reset_retry_policy, -) -from litellm.secret_managers.main import get_secret + _CachingHandlerResponse = None _LLMCachingHandler = None @@ -280,16 +248,7 @@ from typing import ( from openai import OpenAIError as OriginalError -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - update_response_metadata, -) -from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.anthropic_messages.transformation import ( - BaseAnthropicMessagesConfig, -) -from litellm.llms.base_llm.audio_transcription.transformation import ( - BaseAudioTranscriptionConfig, -) +# These are lazy loaded via __getattr__ from litellm.llms.base_llm.base_utils import ( BaseLLMModelInfo, type_to_response_format_param, @@ -340,6 +299,49 @@ if TYPE_CHECKING: from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.mistral.ocr.transformation import MistralOCRConfig + # Type stubs for lazy-loaded functions and classes + from litellm.litellm_core_utils.cached_imports import ( + get_coroutine_checker, + get_litellm_logging_class, + get_set_callbacks, + ) + from litellm.litellm_core_utils.core_helpers import ( + get_litellm_metadata_from_kwargs, + map_finish_reason, + process_response_headers, + ) + from litellm.litellm_core_utils.dot_notation_indexing import ( + delete_nested_value, + is_nested_path, + ) + from litellm.litellm_core_utils.get_litellm_params import ( + _get_base_model_from_litellm_call_metadata, + get_litellm_params, + ) + from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe + from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( + get_formatted_prompt, + ) + from litellm.litellm_core_utils.llm_response_utils.get_headers import ( + get_response_headers, + ) + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, + ) + from litellm.litellm_core_utils.rules import Rules + from litellm.litellm_core_utils.thread_pool_executor import executor + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) + from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig, + ) + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy, + reset_retry_policy, + ) + from litellm.secret_managers.main import get_secret from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig @@ -816,6 +818,7 @@ def function_setup( # noqa: PLR0915 + litellm.failure_callback ) ) + get_set_callbacks = getattr(sys.modules[__name__], 'get_set_callbacks') get_set_callbacks()(callback_list=callback_list, function_id=function_id) ## ASYNC CALLBACKS if len(litellm.input_callback) > 0: @@ -943,6 +946,7 @@ def function_setup( # noqa: PLR0915 elif kwargs.get("messages", None): messages = kwargs["messages"] ### PRE-CALL RULES ### + Rules = getattr(sys.modules[__name__], 'Rules') if ( Rules.has_pre_call_rules() and isinstance(messages, list) @@ -1075,6 +1079,7 @@ def function_setup( # noqa: PLR0915 call_type=call_type, ): stream = True + get_litellm_logging_class = getattr(sys.modules[__name__], 'get_litellm_logging_class') logging_obj = get_litellm_logging_class()( # Victim for object pool model=model, # type: ignore messages=messages, @@ -1158,6 +1163,8 @@ def _get_wrapper_num_retries( if num_retries is None: num_retries = litellm.num_retries if kwargs.get("retry_policy", None): + get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy') + reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy') retry_policy_num_retries = get_num_retries_from_retry_policy( exception=exception, retry_policy=kwargs.get("retry_policy"), @@ -1343,6 +1350,7 @@ def post_call_processing( def client(original_function): # noqa: PLR0915 + Rules = getattr(sys.modules[__name__], 'Rules') rules_obj = Rules() @wraps(original_function) @@ -1528,6 +1536,7 @@ def client(original_function): # noqa: PLR0915 ) else: # RETURN RESULT + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1571,6 +1580,7 @@ def client(original_function): # noqa: PLR0915 # Copy the current context to propagate it to the background thread # This is essential for OpenTelemetry span context propagation ctx = contextvars.copy_context() + executor = getattr(sys.modules[__name__], 'executor') executor.submit( ctx.run, logging_obj.success_handler, @@ -1579,6 +1589,7 @@ def client(original_function): # noqa: PLR0915 end_time, ) # RETURN RESULT + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1595,6 +1606,8 @@ def client(original_function): # noqa: PLR0915 kwargs.get("num_retries", None) or litellm.num_retries or None ) if kwargs.get("retry_policy", None): + get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy') + reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy') num_retries = get_num_retries_from_retry_policy( exception=e, retry_policy=kwargs.get("retry_policy"), @@ -1766,6 +1779,7 @@ def client(original_function): # noqa: PLR0915 chunks, messages=kwargs.get("messages", None) ) else: + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1830,6 +1844,7 @@ def client(original_function): # noqa: PLR0915 end_time=end_time, ) + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1905,6 +1920,7 @@ def client(original_function): # noqa: PLR0915 setattr(e, "timeout", timeout) raise e + get_coroutine_checker = getattr(sys.modules[__name__], 'get_coroutine_checker') is_coroutine = get_coroutine_checker().is_async_callable(original_function) # Return the appropriate wrapper based on the original function type @@ -4448,6 +4464,8 @@ def get_optional_params( # noqa: PLR0915 # Apply nested drops from additional_drop_params if additional_drop_params: + is_nested_path = getattr(sys.modules[__name__], 'is_nested_path') + delete_nested_value = getattr(sys.modules[__name__], 'delete_nested_value') nested_paths = [p for p in additional_drop_params if is_nested_path(p)] for path in nested_paths: optional_params = delete_nested_value(optional_params, path) @@ -4497,6 +4515,7 @@ def add_provider_specific_params_to_optional_params( else: processed_extra_body = initial_extra_body + _ensure_extra_body_is_safe = getattr(sys.modules[__name__], '_ensure_extra_body_is_safe') optional_params["extra_body"] = _ensure_extra_body_is_safe( extra_body=processed_extra_body ) @@ -7022,14 +7041,14 @@ def _get_base_model_from_metadata(model_call_details=None): return _base_model metadata = litellm_params.get("metadata", {}) - base_model_from_metadata = _get_base_model_from_litellm_call_metadata( - metadata=metadata - ) + _get_base_model_from_litellm_call_metadata = getattr(sys.modules[__name__], '_get_base_model_from_litellm_call_metadata') + base_model_from_metadata = _get_base_model_from_litellm_call_metadata(metadata=metadata) if base_model_from_metadata is not None: return base_model_from_metadata # Also check litellm_metadata (used by Responses API and other generic API calls) litellm_metadata = litellm_params.get("litellm_metadata", {}) + _get_base_model_from_litellm_call_metadata = getattr(sys.modules[__name__], '_get_base_model_from_litellm_call_metadata') return _get_base_model_from_litellm_call_metadata(metadata=litellm_metadata) return None @@ -8433,6 +8452,7 @@ def get_end_user_id_for_cost_tracking( service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking. """ + get_litellm_metadata_from_kwargs = getattr(sys.modules[__name__], 'get_litellm_metadata_from_kwargs') _metadata = cast( dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params)) ) @@ -8988,4 +9008,222 @@ def __getattr__(name: str) -> Any: # noqa: PLR0915 _globals["MistralOCRConfig"] = _MistralOCRConfig return _globals["MistralOCRConfig"] + # Lazy load Rules to avoid loading at module import time + if name == "Rules": + # Check if already cached + if "Rules" not in _globals: + from litellm.litellm_core_utils.rules import Rules as _Rules + _globals["Rules"] = _Rules + return _globals["Rules"] + + # Lazy load AsyncHTTPHandler and HTTPHandler to avoid loading at module import time + if name == "AsyncHTTPHandler": + # Check if already cached + if "AsyncHTTPHandler" not in _globals: + from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler as _AsyncHTTPHandler, + ) + _globals["AsyncHTTPHandler"] = _AsyncHTTPHandler + return _globals["AsyncHTTPHandler"] + + if name == "HTTPHandler": + # Check if already cached + if "HTTPHandler" not in _globals: + from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler as _HTTPHandler, + ) + _globals["HTTPHandler"] = _HTTPHandler + return _globals["HTTPHandler"] + + # Lazy load get_num_retries_from_retry_policy and reset_retry_policy to avoid loading at module import time + if name == "get_num_retries_from_retry_policy": + # Check if already cached + if "get_num_retries_from_retry_policy" not in _globals: + from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy, + ) + _globals["get_num_retries_from_retry_policy"] = _get_num_retries_from_retry_policy + return _globals["get_num_retries_from_retry_policy"] + + if name == "reset_retry_policy": + # Check if already cached + if "reset_retry_policy" not in _globals: + from litellm.router_utils.get_retry_from_policy import ( + reset_retry_policy as _reset_retry_policy, + ) + _globals["reset_retry_policy"] = _reset_retry_policy + return _globals["reset_retry_policy"] + + # Lazy load get_secret to avoid loading at module import time + if name == "get_secret": + # Check if already cached + if "get_secret" not in _globals: + from litellm.secret_managers.main import get_secret as _get_secret + _globals["get_secret"] = _get_secret + return _globals["get_secret"] + + # Lazy load cached_imports functions to avoid loading at module import time + if name == "get_coroutine_checker": + # Check if already cached + if "get_coroutine_checker" not in _globals: + from litellm.litellm_core_utils.cached_imports import ( + get_coroutine_checker as _get_coroutine_checker, + ) + _globals["get_coroutine_checker"] = _get_coroutine_checker + return _globals["get_coroutine_checker"] + + if name == "get_litellm_logging_class": + # Check if already cached + if "get_litellm_logging_class" not in _globals: + from litellm.litellm_core_utils.cached_imports import ( + get_litellm_logging_class as _get_litellm_logging_class, + ) + _globals["get_litellm_logging_class"] = _get_litellm_logging_class + return _globals["get_litellm_logging_class"] + + if name == "get_set_callbacks": + # Check if already cached + if "get_set_callbacks" not in _globals: + from litellm.litellm_core_utils.cached_imports import ( + get_set_callbacks as _get_set_callbacks, + ) + _globals["get_set_callbacks"] = _get_set_callbacks + return _globals["get_set_callbacks"] + + # Lazy load core_helpers functions to avoid loading at module import time + if name == "get_litellm_metadata_from_kwargs": + # Check if already cached + if "get_litellm_metadata_from_kwargs" not in _globals: + from litellm.litellm_core_utils.core_helpers import ( + get_litellm_metadata_from_kwargs as _get_litellm_metadata_from_kwargs, + ) + _globals["get_litellm_metadata_from_kwargs"] = _get_litellm_metadata_from_kwargs + return _globals["get_litellm_metadata_from_kwargs"] + + if name == "map_finish_reason": + # Check if already cached + if "map_finish_reason" not in _globals: + from litellm.litellm_core_utils.core_helpers import ( + map_finish_reason as _map_finish_reason, + ) + _globals["map_finish_reason"] = _map_finish_reason + return _globals["map_finish_reason"] + + if name == "process_response_headers": + # Check if already cached + if "process_response_headers" not in _globals: + from litellm.litellm_core_utils.core_helpers import ( + process_response_headers as _process_response_headers, + ) + _globals["process_response_headers"] = _process_response_headers + return _globals["process_response_headers"] + + # Lazy load dot_notation_indexing functions to avoid loading at module import time + if name == "delete_nested_value": + # Check if already cached + if "delete_nested_value" not in _globals: + from litellm.litellm_core_utils.dot_notation_indexing import ( + delete_nested_value as _delete_nested_value, + ) + _globals["delete_nested_value"] = _delete_nested_value + return _globals["delete_nested_value"] + + if name == "is_nested_path": + # Check if already cached + if "is_nested_path" not in _globals: + from litellm.litellm_core_utils.dot_notation_indexing import ( + is_nested_path as _is_nested_path, + ) + _globals["is_nested_path"] = _is_nested_path + return _globals["is_nested_path"] + + # Lazy load get_litellm_params functions to avoid loading at module import time + if name == "_get_base_model_from_litellm_call_metadata": + # Check if already cached + if "_get_base_model_from_litellm_call_metadata" not in _globals: + from litellm.litellm_core_utils.get_litellm_params import ( + _get_base_model_from_litellm_call_metadata as __get_base_model_from_litellm_call_metadata, + ) + _globals["_get_base_model_from_litellm_call_metadata"] = __get_base_model_from_litellm_call_metadata + return _globals["_get_base_model_from_litellm_call_metadata"] + + if name == "get_litellm_params": + # Check if already cached + if "get_litellm_params" not in _globals: + from litellm.litellm_core_utils.get_litellm_params import ( + get_litellm_params as _get_litellm_params, + ) + _globals["get_litellm_params"] = _get_litellm_params + return _globals["get_litellm_params"] + + # Lazy load _ensure_extra_body_is_safe to avoid loading at module import time + if name == "_ensure_extra_body_is_safe": + # Check if already cached + if "_ensure_extra_body_is_safe" not in _globals: + from litellm.litellm_core_utils.llm_request_utils import ( + _ensure_extra_body_is_safe as __ensure_extra_body_is_safe, + ) + _globals["_ensure_extra_body_is_safe"] = __ensure_extra_body_is_safe + return _globals["_ensure_extra_body_is_safe"] + + # Lazy load get_formatted_prompt to avoid loading at module import time + if name == "get_formatted_prompt": + # Check if already cached + if "get_formatted_prompt" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( + get_formatted_prompt as _get_formatted_prompt, + ) + _globals["get_formatted_prompt"] = _get_formatted_prompt + return _globals["get_formatted_prompt"] + + # Lazy load get_response_headers to avoid loading at module import time + if name == "get_response_headers": + # Check if already cached + if "get_response_headers" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.get_headers import ( + get_response_headers as _get_response_headers, + ) + _globals["get_response_headers"] = _get_response_headers + return _globals["get_response_headers"] + + # Lazy load update_response_metadata to avoid loading at module import time + if name == "update_response_metadata": + # Check if already cached + if "update_response_metadata" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata as _update_response_metadata, + ) + _globals["update_response_metadata"] = _update_response_metadata + return _globals["update_response_metadata"] + + # Lazy load executor to avoid loading at module import time + if name == "executor": + # Check if already cached + if "executor" not in _globals: + from litellm.litellm_core_utils.thread_pool_executor import ( + executor as _executor, + ) + _globals["executor"] = _executor + return _globals["executor"] + + # Lazy load BaseAnthropicMessagesConfig to avoid loading at module import time + if name == "BaseAnthropicMessagesConfig": + # Check if already cached + if "BaseAnthropicMessagesConfig" not in _globals: + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig as _BaseAnthropicMessagesConfig, + ) + _globals["BaseAnthropicMessagesConfig"] = _BaseAnthropicMessagesConfig + return _globals["BaseAnthropicMessagesConfig"] + + # Lazy load BaseAudioTranscriptionConfig to avoid loading at module import time + if name == "BaseAudioTranscriptionConfig": + # Check if already cached + if "BaseAudioTranscriptionConfig" not in _globals: + from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig as _BaseAudioTranscriptionConfig, + ) + _globals["BaseAudioTranscriptionConfig"] = _BaseAudioTranscriptionConfig + return _globals["BaseAudioTranscriptionConfig"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From b6d601c2f02d17ae9409366f24320e6cef70f97b Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 16:16:17 -0800 Subject: [PATCH 229/388] perf(utils): lazy load 15+ unused imports (#18616) - Move BaseBatchesConfig, BaseContainerConfig, BaseEmbeddingConfig, BaseImageEditConfig, BaseImageGenerationConfig, BaseImageVariationConfig, BasePassthroughConfig, BaseRealtimeConfig, BaseRerankConfig, BaseVectorStoreConfig, BaseVectorStoreFilesConfig, BaseVideoConfig to lazy loading - Move ANTHROPIC_API_ONLY_HEADERS, AnthropicThinkingParam, RerankResponse to lazy loading - Move ChatCompletionDeltaToolCallChunk, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, LiteLLM_Params to lazy loading - Add type stubs to TYPE_CHECKING block for mypy support - Add lazy loading handlers in __getattr__ method - These imports are not used in utils.py runtime code, only in type annotations (safe with from __future__ import annotations) --- litellm/utils.py | 245 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 216 insertions(+), 29 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 6f4652c8278..df0b2317123 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -149,10 +149,6 @@ def _get_cached_audio_utils(): _audio_utils_module = litellm.litellm_core_utils.audio_utils.utils return _audio_utils_module -from litellm.types.llms.anthropic import ( - ANTHROPIC_API_ONLY_HEADERS, - AnthropicThinkingParam, -) from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -163,7 +159,6 @@ from litellm.types.llms.openai import ( OpenAITextCompletionUserMessage, OpenAIWebSearchOptions, ) -from litellm.types.rerank import RerankResponse from litellm.types.utils import FileTypes # type: ignore from litellm.types.utils import ( OPENAI_RESPONSE_HEADERS, @@ -342,29 +337,41 @@ if TYPE_CHECKING: reset_retry_policy, ) from litellm.secret_managers.main import get_secret + # Type stubs for lazy-loaded config classes and types + from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig + from litellm.llms.base_llm.containers.transformation import BaseContainerConfig + from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig + from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig + from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, + ) + from litellm.llms.base_llm.image_variations.transformation import ( + BaseImageVariationConfig, + ) + from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig + from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig + from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig + from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig + from litellm.llms.base_llm.vector_store_files.transformation import ( + BaseVectorStoreFilesConfig, + ) + from litellm.llms.base_llm.videos.transformation import BaseVideoConfig + from litellm.types.llms.anthropic import ( + ANTHROPIC_API_ONLY_HEADERS, + AnthropicThinkingParam, + ) + from litellm.types.rerank import RerankResponse + from litellm.types.llms.openai import ( + ChatCompletionDeltaToolCallChunk, + ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, + ) + from litellm.types.router import LiteLLM_Params -from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig from litellm.llms.base_llm.completion.transformation import BaseTextCompletionConfig -from litellm.llms.base_llm.containers.transformation import BaseContainerConfig -from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig -from litellm.llms.base_llm.image_generation.transformation import ( - BaseImageGenerationConfig, -) -from litellm.llms.base_llm.image_variations.transformation import ( - BaseImageVariationConfig, -) -from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig -from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.base_llm.skills.transformation import BaseSkillsAPIConfig -from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig -from litellm.llms.base_llm.vector_store_files.transformation import ( - BaseVectorStoreFilesConfig, -) -from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from ._logging import _is_debugging_on, verbose_logger from .caching.caching import ( @@ -392,12 +399,6 @@ from .exceptions import ( UnprocessableEntityError, UnsupportedParamsError, ) -from .types.llms.openai import ( - ChatCompletionDeltaToolCallChunk, - ChatCompletionToolCallChunk, - ChatCompletionToolCallFunctionChunk, -) -from .types.router import LiteLLM_Params if TYPE_CHECKING: from litellm import MockException @@ -9226,4 +9227,190 @@ def __getattr__(name: str) -> Any: # noqa: PLR0915 _globals["BaseAudioTranscriptionConfig"] = _BaseAudioTranscriptionConfig return _globals["BaseAudioTranscriptionConfig"] + # Lazy load BaseBatchesConfig to avoid loading at module import time + if name == "BaseBatchesConfig": + # Check if already cached + if "BaseBatchesConfig" not in _globals: + from litellm.llms.base_llm.batches.transformation import ( + BaseBatchesConfig as _BaseBatchesConfig, + ) + _globals["BaseBatchesConfig"] = _BaseBatchesConfig + return _globals["BaseBatchesConfig"] + + # Lazy load BaseContainerConfig to avoid loading at module import time + if name == "BaseContainerConfig": + # Check if already cached + if "BaseContainerConfig" not in _globals: + from litellm.llms.base_llm.containers.transformation import ( + BaseContainerConfig as _BaseContainerConfig, + ) + _globals["BaseContainerConfig"] = _BaseContainerConfig + return _globals["BaseContainerConfig"] + + # Lazy load BaseEmbeddingConfig to avoid loading at module import time + if name == "BaseEmbeddingConfig": + # Check if already cached + if "BaseEmbeddingConfig" not in _globals: + from litellm.llms.base_llm.embedding.transformation import ( + BaseEmbeddingConfig as _BaseEmbeddingConfig, + ) + _globals["BaseEmbeddingConfig"] = _BaseEmbeddingConfig + return _globals["BaseEmbeddingConfig"] + + # Lazy load BaseImageEditConfig to avoid loading at module import time + if name == "BaseImageEditConfig": + # Check if already cached + if "BaseImageEditConfig" not in _globals: + from litellm.llms.base_llm.image_edit.transformation import ( + BaseImageEditConfig as _BaseImageEditConfig, + ) + _globals["BaseImageEditConfig"] = _BaseImageEditConfig + return _globals["BaseImageEditConfig"] + + # Lazy load BaseImageGenerationConfig to avoid loading at module import time + if name == "BaseImageGenerationConfig": + # Check if already cached + if "BaseImageGenerationConfig" not in _globals: + from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig as _BaseImageGenerationConfig, + ) + _globals["BaseImageGenerationConfig"] = _BaseImageGenerationConfig + return _globals["BaseImageGenerationConfig"] + + # Lazy load BaseImageVariationConfig to avoid loading at module import time + if name == "BaseImageVariationConfig": + # Check if already cached + if "BaseImageVariationConfig" not in _globals: + from litellm.llms.base_llm.image_variations.transformation import ( + BaseImageVariationConfig as _BaseImageVariationConfig, + ) + _globals["BaseImageVariationConfig"] = _BaseImageVariationConfig + return _globals["BaseImageVariationConfig"] + + # Lazy load BasePassthroughConfig to avoid loading at module import time + if name == "BasePassthroughConfig": + # Check if already cached + if "BasePassthroughConfig" not in _globals: + from litellm.llms.base_llm.passthrough.transformation import ( + BasePassthroughConfig as _BasePassthroughConfig, + ) + _globals["BasePassthroughConfig"] = _BasePassthroughConfig + return _globals["BasePassthroughConfig"] + + # Lazy load BaseRealtimeConfig to avoid loading at module import time + if name == "BaseRealtimeConfig": + # Check if already cached + if "BaseRealtimeConfig" not in _globals: + from litellm.llms.base_llm.realtime.transformation import ( + BaseRealtimeConfig as _BaseRealtimeConfig, + ) + _globals["BaseRealtimeConfig"] = _BaseRealtimeConfig + return _globals["BaseRealtimeConfig"] + + # Lazy load BaseRerankConfig to avoid loading at module import time + if name == "BaseRerankConfig": + # Check if already cached + if "BaseRerankConfig" not in _globals: + from litellm.llms.base_llm.rerank.transformation import ( + BaseRerankConfig as _BaseRerankConfig, + ) + _globals["BaseRerankConfig"] = _BaseRerankConfig + return _globals["BaseRerankConfig"] + + # Lazy load BaseVectorStoreConfig to avoid loading at module import time + if name == "BaseVectorStoreConfig": + # Check if already cached + if "BaseVectorStoreConfig" not in _globals: + from litellm.llms.base_llm.vector_store.transformation import ( + BaseVectorStoreConfig as _BaseVectorStoreConfig, + ) + _globals["BaseVectorStoreConfig"] = _BaseVectorStoreConfig + return _globals["BaseVectorStoreConfig"] + + # Lazy load BaseVectorStoreFilesConfig to avoid loading at module import time + if name == "BaseVectorStoreFilesConfig": + # Check if already cached + if "BaseVectorStoreFilesConfig" not in _globals: + from litellm.llms.base_llm.vector_store_files.transformation import ( + BaseVectorStoreFilesConfig as _BaseVectorStoreFilesConfig, + ) + _globals["BaseVectorStoreFilesConfig"] = _BaseVectorStoreFilesConfig + return _globals["BaseVectorStoreFilesConfig"] + + # Lazy load BaseVideoConfig to avoid loading at module import time + if name == "BaseVideoConfig": + # Check if already cached + if "BaseVideoConfig" not in _globals: + from litellm.llms.base_llm.videos.transformation import ( + BaseVideoConfig as _BaseVideoConfig, + ) + _globals["BaseVideoConfig"] = _BaseVideoConfig + return _globals["BaseVideoConfig"] + + # Lazy load ANTHROPIC_API_ONLY_HEADERS to avoid loading at module import time + if name == "ANTHROPIC_API_ONLY_HEADERS": + # Check if already cached + if "ANTHROPIC_API_ONLY_HEADERS" not in _globals: + from litellm.types.llms.anthropic import ( + ANTHROPIC_API_ONLY_HEADERS as _ANTHROPIC_API_ONLY_HEADERS, + ) + _globals["ANTHROPIC_API_ONLY_HEADERS"] = _ANTHROPIC_API_ONLY_HEADERS + return _globals["ANTHROPIC_API_ONLY_HEADERS"] + + # Lazy load AnthropicThinkingParam to avoid loading at module import time + if name == "AnthropicThinkingParam": + # Check if already cached + if "AnthropicThinkingParam" not in _globals: + from litellm.types.llms.anthropic import ( + AnthropicThinkingParam as _AnthropicThinkingParam, + ) + _globals["AnthropicThinkingParam"] = _AnthropicThinkingParam + return _globals["AnthropicThinkingParam"] + + # Lazy load RerankResponse to avoid loading at module import time + if name == "RerankResponse": + # Check if already cached + if "RerankResponse" not in _globals: + from litellm.types.rerank import RerankResponse as _RerankResponse + _globals["RerankResponse"] = _RerankResponse + return _globals["RerankResponse"] + + # Lazy load ChatCompletionDeltaToolCallChunk to avoid loading at module import time + if name == "ChatCompletionDeltaToolCallChunk": + # Check if already cached + if "ChatCompletionDeltaToolCallChunk" not in _globals: + from litellm.types.llms.openai import ( + ChatCompletionDeltaToolCallChunk as _ChatCompletionDeltaToolCallChunk, + ) + _globals["ChatCompletionDeltaToolCallChunk"] = _ChatCompletionDeltaToolCallChunk + return _globals["ChatCompletionDeltaToolCallChunk"] + + # Lazy load ChatCompletionToolCallChunk to avoid loading at module import time + if name == "ChatCompletionToolCallChunk": + # Check if already cached + if "ChatCompletionToolCallChunk" not in _globals: + from litellm.types.llms.openai import ( + ChatCompletionToolCallChunk as _ChatCompletionToolCallChunk, + ) + _globals["ChatCompletionToolCallChunk"] = _ChatCompletionToolCallChunk + return _globals["ChatCompletionToolCallChunk"] + + # Lazy load ChatCompletionToolCallFunctionChunk to avoid loading at module import time + if name == "ChatCompletionToolCallFunctionChunk": + # Check if already cached + if "ChatCompletionToolCallFunctionChunk" not in _globals: + from litellm.types.llms.openai import ( + ChatCompletionToolCallFunctionChunk as _ChatCompletionToolCallFunctionChunk, + ) + _globals["ChatCompletionToolCallFunctionChunk"] = _ChatCompletionToolCallFunctionChunk + return _globals["ChatCompletionToolCallFunctionChunk"] + + # Lazy load LiteLLM_Params to avoid loading at module import time + if name == "LiteLLM_Params": + # Check if already cached + if "LiteLLM_Params" not in _globals: + from litellm.types.router import LiteLLM_Params as _LiteLLM_Params + _globals["LiteLLM_Params"] = _LiteLLM_Params + return _globals["LiteLLM_Params"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From d6296411ec569516a0dabf92ca79b495546146a4 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 16:23:10 -0800 Subject: [PATCH 230/388] SSO Settings Loading, deprecate previous flow --- .../AdminSettings/SSOSettings/SSOSettings.tsx | 106 ++++++++++-------- .../SSOSettingsLoadingSkeleton.tsx | 66 +++++++++++ .../src/components/admins.tsx | 8 +- 3 files changed, 130 insertions(+), 50 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.tsx diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx index d339f6d0e36..27ff96af05f 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettings.tsx @@ -10,12 +10,13 @@ import DeleteSSOSettingsModal from "./Modals/DeleteSSOSettingsModal"; import EditSSOSettingsModal from "./Modals/EditSSOSettingsModal"; import RedactableField from "./RedactableField"; import SSOSettingsEmptyPlaceholder from "./SSOSettingsEmptyPlaceholder"; +import SSOSettingsLoadingSkeleton from "./SSOSettingsLoadingSkeleton"; import { ssoProviderDisplayNames, ssoProviderLogoMap } from "./constants"; const { Title, Text } = Typography; export default function SSOSettings() { - const { data: ssoSettings, refetch } = useSSOSettings(); + const { data: ssoSettings, refetch, isLoading } = useSSOSettings(); const { accessToken } = useAuthorized(); const [isDeleteModalVisible, setIsDeleteModalVisible] = useState(false); const [isAddModalVisible, setIsAddModalVisible] = useState(false); @@ -26,27 +27,28 @@ export default function SSOSettings() { Boolean(ssoSettings?.values.generic_client_id); // Determine the SSO provider based on the configuration - let selectedProvider: string | null = null; - if (ssoSettings?.values.google_client_id) { - selectedProvider = "google"; - } else if (ssoSettings?.values.microsoft_client_id) { - selectedProvider = "microsoft"; - } else if (ssoSettings?.values.generic_client_id) { - // Check if it looks like Okta based on endpoints - if ( - ssoSettings.values.generic_authorization_endpoint?.includes("okta") || - ssoSettings.values.generic_authorization_endpoint?.includes("auth0") - ) { - selectedProvider = "okta"; - } else { - selectedProvider = "generic"; + const detectSSOProvider = (values: SSOSettingsValues): string | null => { + if (values.google_client_id) return "google"; + if (values.microsoft_client_id) return "microsoft"; + if (values.generic_client_id) { + // Check if it looks like Okta/Auth0 based on endpoints + if ( + values.generic_authorization_endpoint?.includes("okta") || + values.generic_authorization_endpoint?.includes("auth0") + ) { + return "okta"; + } + return "generic"; } - } + return null; + }; + + const selectedProvider = ssoSettings?.values ? detectSSOProvider(ssoSettings.values) : null; const renderEndpointValue = (value?: string | null) => ( - - {value || Not configured} - + + {value || "-"} + ); const renderSimpleValue = (value?: string | null) => @@ -179,38 +181,44 @@ export default function SSOSettings() { }; return ( - - - {/* Header Section */} -
-
- -
- SSO Configuration - Manage Single Sign-On authentication settings + <> + {isLoading ? ( + + ) : ( + + + {/* Header Section */} +
+
+ +
+ SSO Configuration + Manage Single Sign-On authentication settings +
+
+ +
+ {isSSOConfigured && ( + <> + + + + )} +
-
-
- {isSSOConfigured && ( - <> - - - + {isSSOConfigured ? ( + renderSSOSettings() + ) : ( + setIsAddModalVisible(true)} /> )} -
-
- - {isSSOConfigured ? ( - renderSSOSettings() - ) : ( - setIsAddModalVisible(true)} /> - )} - + + + )} - + ); } diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.tsx new file mode 100644 index 00000000000..59e34f255e3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.tsx @@ -0,0 +1,66 @@ +"use client"; + +import { Card, Descriptions, Skeleton, Space, Typography } from "antd"; +import { Shield } from "lucide-react"; + +const { Title, Text } = Typography; +export default function SSOSettingsLoadingSkeleton() { + const descriptionsConfig = { + column: { + xxl: 1, + xl: 1, + lg: 1, + md: 1, + sm: 1, + xs: 1, + }, + }; + + return ( + + + {/* Header Section */} +
+
+ +
+ SSO Configuration + Manage Single Sign-On authentication settings +
+
+ +
+ + +
+
+ + {/* Descriptions Table Skeleton */} + + {/* Provider Row */} + }> +
+ +
+ + + }> + + + + }> + + + + }> + + + + }> + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/components/admins.tsx b/ui/litellm-dashboard/src/components/admins.tsx index 6af5a226da6..9de971bcd62 100644 --- a/ui/litellm-dashboard/src/components/admins.tsx +++ b/ui/litellm-dashboard/src/components/admins.tsx @@ -3,7 +3,7 @@ * Use this to avoid sharing master key with others */ import React, { useState, useEffect } from "react"; -import { Typography } from "antd"; +import { Alert, Typography } from "antd"; import { useRouter } from "next/navigation"; import { Button as Button2, Modal, Form, Input } from "antd"; import { Select, SelectItem } from "@tremor/react"; @@ -509,6 +509,12 @@ const AdminPanel: React.FC = ({ ✨ Security Settings +
Date: Sat, 3 Jan 2026 17:25:46 -0800 Subject: [PATCH 231/388] Adding unit testing coverage --- .../Modals/EditSSOSettingsModal.test.tsx | 620 ++++++++++++++++++ .../SSOSettingsLoadingSkeleton.test.tsx | 222 +++++++ .../VectorStoreSelector.test.tsx | 524 +++++++++++++++ .../VectorStoreTable.test.tsx | 415 ++++++++++++ .../src/utils/cookieUtils.test.ts | 72 ++ .../src/utils/proxyUtils.test.ts | 78 +++ .../src/utils/textUtils.test.ts | 21 + 7 files changed, 1952 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.test.tsx create mode 100644 ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.test.tsx create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx create mode 100644 ui/litellm-dashboard/src/utils/proxyUtils.test.ts diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.test.tsx new file mode 100644 index 00000000000..559d837b409 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.test.tsx @@ -0,0 +1,620 @@ +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import { describe, it, expect, vi, beforeEach, Mock } from "vitest"; +import EditSSOSettingsModal from "./EditSSOSettingsModal"; +import { useSSOSettings } from "@/app/(dashboard)/hooks/sso/useSSOSettings"; +import { useEditSSOSettings } from "@/app/(dashboard)/hooks/sso/useEditSSOSettings"; +import NotificationsManager from "@/components/molecules/notifications_manager"; +import { parseErrorMessage } from "@/components/shared/errorUtils"; +import { processSSOSettingsPayload } from "../utils"; + +// Constants +const SSO_PROVIDERS = { + GOOGLE: "google", + MICROSOFT: "microsoft", + OKTA: "okta", + AUTH0: "auth0", + GENERIC: "generic", +} as const; + +const TEST_DATA = { + MODAL_TITLE: "Edit SSO Settings", + MODAL_WIDTH: "800", + SUCCESS_MESSAGE: "SSO settings updated successfully", + ERROR_MESSAGE_PREFIX: "Failed to save SSO settings:", + BUTTON_TEXT: { + CANCEL: "Cancel", + SAVE: "Save", + SAVING: "Saving...", + }, +} as const; + +const TEST_IDS = { + MODAL: "modal", + BUTTON: "button", + BASE_SSO_FORM: "base-sso-form", + TRIGGER_FORM_SUBMIT: "trigger-form-submit", +} as const; + +// Mock form instance +const mockForm = { + resetFields: vi.fn(), + setFieldsValue: vi.fn(), + getFieldsValue: vi.fn(), + submit: vi.fn(), +}; + +// Types +type SSOData = { + values: Record; +} & Record; + +type SSOSettingsHookReturn = { + data: SSOData | null; + isLoading: boolean; + error: any; +}; + +type EditSSOSettingsHookReturn = { + mutateAsync: ReturnType; + isPending: boolean; +}; + +// Test data factories +const createSSOData = (overrides: Record = {}): SSOData => ({ + values: { + user_email: "test@example.com", + ...overrides, + }, +}); + +const createGoogleSSOData = (overrides: Record = {}) => + createSSOData({ + google_client_id: "test-google-id", + google_client_secret: "test-google-secret", + ...overrides, + }); + +const createMicrosoftSSOData = (overrides: Record = {}) => + createSSOData({ + microsoft_client_id: "test-microsoft-id", + microsoft_client_secret: "test-microsoft-secret", + microsoft_tenant: "test-tenant", + ...overrides, + }); + +const createGenericSSOData = (overrides: Record = {}) => + createSSOData({ + generic_client_id: "test-generic-id", + generic_client_secret: "test-generic-secret", + generic_authorization_endpoint: overrides.authorization_endpoint || "https://custom.example.com/oauth", + ...overrides, + }); + +const createRoleMappingsSSOData = (overrides: Record = {}) => + createGoogleSSOData({ + role_mappings: { + group_claim: "groups", + default_role: "internal_user", + roles: { + proxy_admin: overrides.proxy_admin || ["admin-group"], + proxy_admin_viewer: overrides.proxy_admin_viewer || ["viewer-group"], + internal_user: overrides.internal_user || ["user-group"], + internal_user_viewer: overrides.internal_user_viewer || ["readonly-group"], + }, + }, + ...overrides, + }); + +// Mock utilities +const createMockHooks = (): { + useSSOSettings: SSOSettingsHookReturn; + useEditSSOSettings: EditSSOSettingsHookReturn; +} => ({ + useSSOSettings: { + data: null, + isLoading: false, + error: null, + }, + useEditSSOSettings: { + mutateAsync: vi.fn(), + isPending: false, + }, +}); + +vi.mock("antd", () => ({ + Modal: ({ children, open, title, footer, onCancel, width, ...props }: any) => ( +
+
{children}
+
{footer}
+
+ ), + Button: ({ children, onClick, loading, disabled, ...props }: any) => ( + + ), + Form: { + useForm: () => [mockForm], + }, + Space: ({ children, ...props }: any) => ( +
+ {children} +
+ ), +})); + +vi.mock("./BaseSSOSettingsForm", () => ({ + default: ({ form, onFormSubmit }: any) => ( +
+ +
+ ), +})); + +vi.mock("@/app/(dashboard)/hooks/sso/useSSOSettings", () => ({ + useSSOSettings: vi.fn(), +})); + +vi.mock("@/app/(dashboard)/hooks/sso/useEditSSOSettings", () => ({ + useEditSSOSettings: vi.fn(), +})); + +vi.mock("@/components/molecules/notifications_manager", () => ({ + default: { + success: vi.fn(), + fromBackend: vi.fn(), + }, +})); + +vi.mock("@/components/shared/errorUtils", () => ({ + parseErrorMessage: vi.fn(), +})); + +vi.mock("../utils", () => ({ + processSSOSettingsPayload: vi.fn(), +})); + +// Test helpers +const setupMocks = ( + overrides: Partial<{ + useSSOSettings: Partial; + useEditSSOSettings: Partial; + }> = {}, +) => { + const defaultMocks = createMockHooks(); + const mocks = { + useSSOSettings: { ...defaultMocks.useSSOSettings, ...overrides.useSSOSettings }, + useEditSSOSettings: { ...defaultMocks.useEditSSOSettings, ...overrides.useEditSSOSettings }, + }; + + (useSSOSettings as Mock).mockReturnValue(mocks.useSSOSettings); + (useEditSSOSettings as Mock).mockReturnValue(mocks.useEditSSOSettings); + + return mocks; +}; + +const renderComponent = (props: Partial> = {}) => { + const defaultProps = { + isVisible: true, + onCancel: vi.fn(), + onSuccess: vi.fn(), + }; + + return { + ...render(), + mockOnCancel: defaultProps.onCancel, + mockOnSuccess: defaultProps.onSuccess, + }; +}; + +const getButtons = () => screen.getAllByTestId(TEST_IDS.BUTTON); +const getCancelButton = () => getButtons()[0]; +const getSaveButton = () => getButtons()[1]; + +describe("EditSSOSettingsModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + setupMocks(); + }); + + describe("Rendering", () => { + it("renders without crashing", () => { + expect(() => renderComponent()).not.toThrow(); + }); + + it("displays modal with correct configuration", () => { + renderComponent(); + + const modal = screen.getByTestId(TEST_IDS.MODAL); + expect(modal).toHaveAttribute("data-open", "true"); + expect(modal).toHaveAttribute("data-title", TEST_DATA.MODAL_TITLE); + expect(modal).toHaveAttribute("data-width", TEST_DATA.MODAL_WIDTH); + }); + + it("displays modal as closed when not visible", () => { + renderComponent({ isVisible: false }); + + const modal = screen.getByTestId(TEST_IDS.MODAL); + expect(modal).toHaveAttribute("data-open", "false"); + }); + }); + + describe("Footer Actions", () => { + it("renders cancel and save buttons", () => { + renderComponent(); + + const buttons = getButtons(); + expect(buttons).toHaveLength(2); + expect(buttons[0]).toHaveTextContent(TEST_DATA.BUTTON_TEXT.CANCEL); + expect(buttons[1]).toHaveTextContent(TEST_DATA.BUTTON_TEXT.SAVE); + }); + + it("calls onCancel and resets form when cancel button is clicked", () => { + const { mockOnCancel } = renderComponent(); + + fireEvent.click(getCancelButton()); + + expect(mockForm.resetFields).toHaveBeenCalled(); + expect(mockOnCancel).toHaveBeenCalled(); + }); + + it("calls form.submit when save button is clicked", () => { + renderComponent(); + + fireEvent.click(getSaveButton()); + + expect(mockForm.submit).toHaveBeenCalled(); + }); + + describe("Loading States", () => { + it("disables cancel button during submission", () => { + setupMocks({ + useEditSSOSettings: { mutateAsync: vi.fn(), isPending: true }, + }); + + renderComponent(); + + expect(getCancelButton()).toBeDisabled(); + }); + + it("shows loading state on save button during submission", () => { + setupMocks({ + useEditSSOSettings: { mutateAsync: vi.fn(), isPending: true }, + }); + + renderComponent(); + + expect(getSaveButton()).toHaveAttribute("data-loading", "true"); + expect(getSaveButton()).toHaveTextContent(TEST_DATA.BUTTON_TEXT.SAVING); + }); + }); + }); + + describe("Form Submission", () => { + const formValues = { testField: "testValue" }; + const processedPayload = { processed: "payload" }; + + beforeEach(() => { + (processSSOSettingsPayload as any).mockReturnValue(processedPayload); + }); + + it("processes form values and submits successfully", async () => { + const mockMutateAsync = vi.fn().mockImplementation((payload, options) => { + options.onSuccess(); + return Promise.resolve({ success: true }); + }); + + setupMocks({ + useEditSSOSettings: { mutateAsync: mockMutateAsync, isPending: false }, + }); + + const { mockOnSuccess } = renderComponent(); + + fireEvent.click(screen.getByTestId(TEST_IDS.TRIGGER_FORM_SUBMIT)); + + expect(processSSOSettingsPayload).toHaveBeenCalledWith(formValues); + expect(mockMutateAsync).toHaveBeenCalledWith( + processedPayload, + expect.objectContaining({ + onSuccess: expect.any(Function), + onError: expect.any(Function), + }), + ); + }); + + it("shows success notification and calls onSuccess callback", async () => { + const mockMutateAsync = vi.fn().mockImplementation((payload, options) => { + options.onSuccess(); + return Promise.resolve({ success: true }); + }); + + setupMocks({ + useEditSSOSettings: { mutateAsync: mockMutateAsync, isPending: false }, + }); + + const { mockOnSuccess } = renderComponent(); + + fireEvent.click(screen.getByTestId(TEST_IDS.TRIGGER_FORM_SUBMIT)); + + expect(NotificationsManager.success).toHaveBeenCalledWith(TEST_DATA.SUCCESS_MESSAGE); + expect(mockOnSuccess).toHaveBeenCalled(); + }); + + it("handles submission errors gracefully", async () => { + const error = new Error("Submission failed"); + const mockMutateAsync = vi.fn().mockImplementation((payload, options) => { + options.onError(error); + return Promise.reject(error); + }); + + setupMocks({ + useEditSSOSettings: { mutateAsync: mockMutateAsync, isPending: false }, + }); + + (parseErrorMessage as any).mockReturnValue("Parsed error message"); + + renderComponent(); + + fireEvent.click(screen.getByTestId(TEST_IDS.TRIGGER_FORM_SUBMIT)); + + expect(parseErrorMessage).toHaveBeenCalledWith(error); + expect(NotificationsManager.fromBackend).toHaveBeenCalledWith( + `${TEST_DATA.ERROR_MESSAGE_PREFIX} Parsed error message`, + ); + }); + }); + + describe("Form Initialization", () => { + describe("Provider Detection", () => { + const testProviderDetection = (testName: string, ssoData: SSOData, expectedProvider: string) => { + it(`detects ${testName} provider`, async () => { + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent(); + + await waitFor(() => { + expect(mockForm.setFieldsValue).toHaveBeenCalledWith({ + sso_provider: expectedProvider, + ...ssoData.values, + }); + }); + }); + }; + + testProviderDetection("Google", createGoogleSSOData(), SSO_PROVIDERS.GOOGLE); + + testProviderDetection("Microsoft", createMicrosoftSSOData(), SSO_PROVIDERS.MICROSOFT); + + testProviderDetection( + "Okta", + createGenericSSOData({ + authorization_endpoint: "https://okta.example.com/oauth2/authorize", + }), + SSO_PROVIDERS.OKTA, + ); + + testProviderDetection( + "Auth0 (detected as Okta)", + createGenericSSOData({ + authorization_endpoint: "https://auth0.example.com/authorize", + }), + SSO_PROVIDERS.OKTA, // Auth0 URLs are detected as Okta provider + ); + + testProviderDetection("generic", createGenericSSOData(), SSO_PROVIDERS.GENERIC); + }); + + describe("Role Mappings", () => { + it("processes role mappings with all roles assigned", async () => { + const ssoData = createRoleMappingsSSOData(); + + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent(); + + await waitFor(() => { + expect(mockForm.setFieldsValue).toHaveBeenCalledWith({ + sso_provider: SSO_PROVIDERS.GOOGLE, + ...ssoData.values, + use_role_mappings: true, + group_claim: "groups", + default_role: "internal_user", + proxy_admin_teams: "admin-group", + admin_viewer_teams: "viewer-group", + internal_user_teams: "user-group", + internal_viewer_teams: "readonly-group", + }); + }); + }); + + it("handles empty role mapping arrays", async () => { + const ssoData = createRoleMappingsSSOData({ + proxy_admin: [], + proxy_admin_viewer: [], + internal_user_viewer: [], + }); + + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent(); + + await waitFor(() => { + expect(mockForm.setFieldsValue).toHaveBeenCalledWith({ + sso_provider: SSO_PROVIDERS.GOOGLE, + ...ssoData.values, + use_role_mappings: true, + group_claim: "groups", + default_role: "internal_user", + proxy_admin_teams: "", + admin_viewer_teams: "", + internal_user_teams: "user-group", + internal_viewer_teams: "", + }); + }); + }); + }); + + describe("Initialization Guards", () => { + it("resets form before setting values", async () => { + const ssoData = createGoogleSSOData(); + + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent(); + + await waitFor(() => { + expect(mockForm.resetFields).toHaveBeenCalled(); + expect(mockForm.setFieldsValue).toHaveBeenCalled(); + }); + }); + + it("skips initialization when modal is not visible", () => { + const ssoData = createGoogleSSOData(); + + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent({ isVisible: false }); + + expect(mockForm.setFieldsValue).not.toHaveBeenCalled(); + }); + + it("skips initialization when SSO data is unavailable", () => { + setupMocks({ + useSSOSettings: { data: null, isLoading: false, error: null }, + }); + + renderComponent(); + + expect(mockForm.setFieldsValue).not.toHaveBeenCalled(); + }); + }); + }); + + describe("Error Handling", () => { + it("handles form submission errors with undefined error message", async () => { + const error = new Error("Network error"); + const mockMutateAsync = vi.fn().mockImplementation((payload, options) => { + options.onError(error); + return Promise.reject(error); + }); + + setupMocks({ + useEditSSOSettings: { mutateAsync: mockMutateAsync, isPending: false }, + }); + + (parseErrorMessage as any).mockReturnValue(undefined); + + renderComponent(); + + fireEvent.click(screen.getByTestId(TEST_IDS.TRIGGER_FORM_SUBMIT)); + + expect(NotificationsManager.fromBackend).toHaveBeenCalledWith(`${TEST_DATA.ERROR_MESSAGE_PREFIX} undefined`); + }); + + it("handles form submission with malformed data", async () => { + const mockMutateAsync = vi.fn().mockImplementation((payload, options) => { + options.onError(new Error("Invalid data")); + return Promise.reject(new Error("Invalid data")); + }); + + setupMocks({ + useEditSSOSettings: { mutateAsync: mockMutateAsync, isPending: false }, + }); + + (processSSOSettingsPayload as any).mockImplementation(() => { + throw new Error("Processing failed"); + }); + + renderComponent(); + + fireEvent.click(screen.getByTestId(TEST_IDS.TRIGGER_FORM_SUBMIT)); + + expect(processSSOSettingsPayload).toHaveBeenCalled(); + expect(mockMutateAsync).not.toHaveBeenCalled(); + }); + }); + + describe("Edge Cases", () => { + it("handles role mappings with undefined roles object", async () => { + const ssoData = createGoogleSSOData({ + role_mappings: { + group_claim: "groups", + default_role: "internal_user", + // roles is undefined + }, + }); + + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent(); + + await waitFor(() => { + expect(mockForm.setFieldsValue).toHaveBeenCalledWith({ + sso_provider: SSO_PROVIDERS.GOOGLE, + ...ssoData.values, + use_role_mappings: true, + group_claim: "groups", + default_role: "internal_user", + proxy_admin_teams: "", + admin_viewer_teams: "", + internal_user_teams: "", + internal_viewer_teams: "", + }); + }); + }); + + it("handles provider detection with partial SSO data", async () => { + const ssoData = createSSOData({ + // Only has generic fields, no specific provider identifiers + generic_client_id: "test-id", + generic_authorization_endpoint: "https://unknown.provider.com/auth", + }); + + setupMocks({ + useSSOSettings: { data: ssoData, isLoading: false, error: null }, + }); + + renderComponent(); + + await waitFor(() => { + expect(mockForm.setFieldsValue).toHaveBeenCalledWith({ + sso_provider: SSO_PROVIDERS.GENERIC, + ...ssoData.values, + }); + }); + }); + + it("handles form submission when processing throws error", async () => { + setupMocks({ + useEditSSOSettings: { mutateAsync: vi.fn(), isPending: false }, + }); + + (processSSOSettingsPayload as any).mockImplementation(() => { + throw new Error("Processing error"); + }); + + renderComponent(); + + expect(() => { + fireEvent.click(screen.getByTestId(TEST_IDS.TRIGGER_FORM_SUBMIT)); + }).not.toThrow(); + + expect(processSSOSettingsPayload).toHaveBeenCalled(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.test.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.test.tsx new file mode 100644 index 00000000000..fd4fde69588 --- /dev/null +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/SSOSettingsLoadingSkeleton.test.tsx @@ -0,0 +1,222 @@ +import { render, screen } from "@testing-library/react"; +import { describe, it, expect, vi } from "vitest"; +import SSOSettingsLoadingSkeleton from "./SSOSettingsLoadingSkeleton"; + +// Mock lucide-react icons +vi.mock("lucide-react", () => ({ + Shield: ({ className }: any) =>
, +})); + +// Mock Ant Design components +vi.mock("antd", () => ({ + Card: ({ children, ...props }: any) => ( +
+ {children} +
+ ), + Descriptions: Object.assign( + ({ children, bordered, column, ...props }: any) => ( +
+ {children} +
+ ), + { + Item: ({ children, label, ...props }: any) => ( +
+
{label}
+
{children}
+
+ ), + }, + ), + Typography: { + Title: ({ children, level, ...props }: any) => ( +
+ {children} +
+ ), + Text: ({ children, type, ...props }: any) => ( +
+ {children} +
+ ), + }, + Space: ({ children, direction, size, className, ...props }: any) => ( +
+ {children} +
+ ), + Skeleton: { + Button: ({ active, size, style, ...props }: any) => ( +
+ Button Skeleton +
+ ), + Node: ({ active, style, ...props }: any) => ( +
+ Node Skeleton +
+ ), + }, +})); + +describe("SSOSettingsLoadingSkeleton", () => { + it("should render without crashing", () => { + expect(() => render()).not.toThrow(); + }); + + it("should render Card component", () => { + render(); + expect(screen.getByTestId("card")).toBeInTheDocument(); + }); + + it("should render Space component with correct props", () => { + render(); + const space = screen.getByTestId("space"); + expect(space).toBeInTheDocument(); + expect(space).toHaveAttribute("data-direction", "vertical"); + expect(space).toHaveAttribute("data-size", "large"); + expect(space).toHaveClass("w-full"); + }); + + describe("Header Section", () => { + it("should render Shield icon", () => { + render(); + const shieldIcon = screen.getByTestId("shield-icon"); + expect(shieldIcon).toBeInTheDocument(); + expect(shieldIcon).toHaveClass("w-6 h-6 text-gray-400"); + }); + + it("should render title with correct text and level", () => { + render(); + const title = screen.getByTestId("typography-title"); + expect(title).toBeInTheDocument(); + expect(title).toHaveAttribute("data-level", "3"); + expect(title).toHaveTextContent("SSO Configuration"); + }); + + it("should render subtitle text", () => { + render(); + const text = screen.getByTestId("typography-text"); + expect(text).toBeInTheDocument(); + expect(text).toHaveAttribute("data-type", "secondary"); + expect(text).toHaveTextContent("Manage Single Sign-On authentication settings"); + }); + + it("should render two skeleton buttons with correct styles", () => { + render(); + const buttons = screen.getAllByTestId("skeleton-button"); + expect(buttons).toHaveLength(2); + + // First button + expect(buttons[0]).toHaveAttribute("data-active", "true"); + expect(buttons[0]).toHaveAttribute("data-size", "default"); + expect(buttons[0]).toHaveAttribute("data-style", JSON.stringify({ width: 170, height: 32 })); + + // Second button + expect(buttons[1]).toHaveAttribute("data-active", "true"); + expect(buttons[1]).toHaveAttribute("data-size", "default"); + expect(buttons[1]).toHaveAttribute("data-style", JSON.stringify({ width: 190, height: 32 })); + }); + }); + + describe("Descriptions Table", () => { + it("should render Descriptions component with bordered prop", () => { + render(); + const descriptions = screen.getByTestId("descriptions"); + expect(descriptions).toBeInTheDocument(); + expect(descriptions).toHaveAttribute("data-bordered", "true"); + }); + + it("should apply correct column configuration", () => { + render(); + const descriptions = screen.getByTestId("descriptions"); + const expectedColumn = { + xxl: 1, + xl: 1, + lg: 1, + md: 1, + sm: 1, + xs: 1, + }; + expect(descriptions).toHaveAttribute("data-column", JSON.stringify(expectedColumn)); + }); + + it("should render exactly 5 description items", () => { + render(); + const items = screen.getAllByTestId("descriptions-item"); + expect(items).toHaveLength(5); + }); + + describe("Description Items Structure", () => { + it("should render exactly 10 skeleton nodes total", () => { + render(); + const skeletonNodes = screen.getAllByTestId("skeleton-node"); + expect(skeletonNodes).toHaveLength(10); + }); + + it("should render 5 skeleton nodes for labels with width 80", () => { + render(); + const skeletonNodes = screen.getAllByTestId("skeleton-node"); + + const labelNodes = skeletonNodes.filter( + (node) => node.getAttribute("data-style") === JSON.stringify({ width: 80, height: 16 }), + ); + expect(labelNodes).toHaveLength(5); + + labelNodes.forEach((node) => { + expect(node).toHaveAttribute("data-active", "true"); + }); + }); + + it("should render skeleton nodes for content with correct widths", () => { + render(); + const skeletonNodes = screen.getAllByTestId("skeleton-node"); + + // Expected content widths: [100, 200, 250, 180, 220] + const expectedWidths = [100, 200, 250, 180, 220]; + expectedWidths.forEach((width) => { + const contentNode = skeletonNodes.find( + (node) => node.getAttribute("data-style") === JSON.stringify({ width, height: 16 }), + ); + expect(contentNode).toBeInTheDocument(); + expect(contentNode).toHaveAttribute("data-active", "true"); + }); + }); + }); + }); + + describe("Accessibility and Structure", () => { + it("should have proper semantic structure", () => { + render(); + // Card contains Space + const card = screen.getByTestId("card"); + const space = screen.getByTestId("space"); + expect(card).toContainElement(space); + + // Space contains header section and descriptions + const descriptions = screen.getByTestId("descriptions"); + expect(space).toContainElement(descriptions); + }); + + it("should render all skeleton elements as active", () => { + render(); + const skeletonNodes = screen.getAllByTestId("skeleton-node"); + const skeletonButtons = screen.getAllByTestId("skeleton-button"); + + skeletonNodes.forEach((node) => { + expect(node).toHaveAttribute("data-active", "true"); + }); + + skeletonButtons.forEach((button) => { + expect(button).toHaveAttribute("data-active", "true"); + }); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx new file mode 100644 index 00000000000..8c6b85a53de --- /dev/null +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx @@ -0,0 +1,524 @@ +import { render, screen, waitFor, fireEvent } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import VectorStoreSelector from "./VectorStoreSelector"; +import { vectorStoreListCall } from "../networking"; +import { VectorStore } from "./types"; + +// Mock dependencies +const mockVectorStoreListCall = vi.fn(); + +vi.mock("../networking", () => ({ + vectorStoreListCall: (...args: any[]) => mockVectorStoreListCall(...args), +})); + +// Mock antd Select component +vi.mock("antd", () => ({ + Select: vi.fn(), +})); + +// Import the mocked Select +import { Select as MockedSelect } from "antd"; + +// Configure the mock to render a simple div with data attributes +(MockedSelect as any).mockImplementation((props: any) => { + const { + onChange, + value, + placeholder, + loading, + className, + disabled, + options, + mode, + showSearch, + optionFilterProp, + style, + } = props; + + return ( +
{ + // For testing purposes, allow simulating different selection behaviors + // The test can control this by setting data attributes on the element + const testSelection = e.target.getAttribute("data-test-selection"); + if (testSelection && onChange) { + onChange(JSON.parse(testSelection)); + } else if (onChange && options?.length > 0) { + // Default behavior: select first option + onChange([options[0].value]); + } + }} + > + {options?.map((opt: any) => ( +
+ {opt.label} +
+ ))} +
+ ); +}); + +// Test helpers +const mockOnChange = vi.fn(); +const mockAccessToken = "test-token"; + +const mockVectorStores: VectorStore[] = [ + { + vector_store_id: "store-1", + custom_llm_provider: "openai", + vector_store_name: "My Store", + vector_store_description: "A test store", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }, + { + vector_store_id: "store-2", + custom_llm_provider: "azure", + vector_store_name: "Another Store", + vector_store_description: "Another test store", + created_at: "2024-01-02T00:00:00Z", + updated_at: "2024-01-02T00:00:00Z", + }, + { + vector_store_id: "store-3", + custom_llm_provider: "pg_vector", + // No vector_store_name to test fallback to vector_store_id + vector_store_description: "Store without name", + created_at: "2024-01-03T00:00:00Z", + updated_at: "2024-01-03T00:00:00Z", + }, +]; + +const defaultProps = { + onChange: mockOnChange, + accessToken: mockAccessToken, +}; + +// Helper functions +const renderComponent = (props = {}) => { + return render(); +}; + +const waitForDataFetch = async () => { + await waitFor(() => { + expect(mockVectorStoreListCall).toHaveBeenCalled(); + }); +}; + +const getSelectElement = () => screen.getByTestId("vector-store-select"); + +const getOptionElements = () => + screen.getAllByTestId(/^vector-store-select/).filter((el) => el.hasAttribute("data-option-value")); + +describe("VectorStoreSelector", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockVectorStoreListCall.mockResolvedValue({ + data: mockVectorStores, + }); + }); + + describe("Rendering", () => { + it("should render the select component", () => { + renderComponent(); + expect(getSelectElement()).toBeInTheDocument(); + }); + + it("should render with default placeholder", () => { + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-placeholder", "Select vector stores"); + }); + + it("should render with custom placeholder", () => { + renderComponent({ placeholder: "Choose stores" }); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-placeholder", "Choose stores"); + }); + + it("should apply custom className", () => { + renderComponent({ className: "custom-class" }); + const select = getSelectElement(); + expect(select).toHaveClass("custom-class"); + }); + + it("should render with disabled state", () => { + renderComponent({ disabled: true }); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-disabled", "true"); + }); + + it("should render with enabled state by default", () => { + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-disabled", "false"); + }); + + it("should render with multiple mode", () => { + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-mode", "multiple"); + }); + + it("should render with showSearch enabled", () => { + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-show-search", "true"); + }); + + it("should render with optionFilterProp set to label", () => { + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-option-filter-prop", "label"); + }); + + it("should render with full width style", () => { + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveStyle({ width: "100%" }); + }); + }); + + describe("Data fetching", () => { + it("should fetch vector stores on mount when accessToken is provided", async () => { + renderComponent(); + await waitFor(() => { + expect(mockVectorStoreListCall).toHaveBeenCalledWith(mockAccessToken); + }); + }); + + it("should not fetch vector stores when accessToken is falsy", () => { + const { rerender } = render(); + expect(mockVectorStoreListCall).not.toHaveBeenCalled(); + + rerender(); + expect(mockVectorStoreListCall).not.toHaveBeenCalled(); + + rerender(); + expect(mockVectorStoreListCall).not.toHaveBeenCalled(); + }); + + it("should fetch vector stores again when accessToken changes", async () => { + const { rerender } = render(); + await waitFor(() => { + expect(mockVectorStoreListCall).toHaveBeenCalledWith("token-1"); + }); + + vi.clearAllMocks(); + rerender(); + await waitFor(() => { + expect(mockVectorStoreListCall).toHaveBeenCalledWith("token-2"); + }); + }); + + it("should set loading state while fetching", async () => { + let resolvePromise: (value: any) => void; + const promise = new Promise((resolve) => { + resolvePromise = resolve; + }); + mockVectorStoreListCall.mockReturnValue(promise); + + renderComponent(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-loading", "true"); + + resolvePromise!({ data: mockVectorStores }); + await waitFor(() => { + expect(select).toHaveAttribute("data-loading", "false"); + }); + }); + + it("should clear loading state after successful fetch", async () => { + renderComponent(); + await waitForDataFetch(); + const select = getSelectElement(); + expect(select).toHaveAttribute("data-loading", "false"); + }); + + it("should clear loading state after failed fetch", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mockVectorStoreListCall.mockRejectedValueOnce(new Error("Network error")); + + renderComponent(); + await waitForDataFetch(); + + const select = getSelectElement(); + expect(select).toHaveAttribute("data-loading", "false"); + consoleErrorSpy.mockRestore(); + }); + }); + + describe("Options rendering", () => { + it("should render vector store options after successful fetch", async () => { + renderComponent(); + await waitForDataFetch(); + + expect(screen.getByText("My Store (store-1)")).toBeInTheDocument(); + expect(screen.getByText("Another Store (store-2)")).toBeInTheDocument(); + expect(screen.getByText("store-3 (store-3)")).toBeInTheDocument(); + }); + + it("should use vector_store_name when available for label", async () => { + renderComponent(); + await waitForDataFetch(); + + const option1 = screen.getByText("My Store (store-1)"); + expect(option1).toBeInTheDocument(); + expect(option1).toHaveAttribute("data-option-title", "A test store"); + }); + + it("should fallback to vector_store_id when vector_store_name is missing", async () => { + renderComponent(); + await waitForDataFetch(); + + const option3 = screen.getByText("store-3 (store-3)"); + expect(option3).toBeInTheDocument(); + // When vector_store_name is missing, title uses vector_store_description if available, otherwise vector_store_id + expect(option3).toHaveAttribute("data-option-title", "Store without name"); + }); + + it("should use vector_store_description as title when available", async () => { + renderComponent(); + await waitForDataFetch(); + + const option1 = screen.getByText("My Store (store-1)"); + expect(option1).toHaveAttribute("data-option-title", "A test store"); + }); + + it("should fallback to vector_store_id as title when vector_store_description is missing", async () => { + const storesWithoutDescription: VectorStore[] = [ + { + vector_store_id: "store-no-desc", + custom_llm_provider: "openai", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }, + ]; + mockVectorStoreListCall.mockResolvedValueOnce({ + data: storesWithoutDescription, + }); + + renderComponent(); + await waitForDataFetch(); + + const option = screen.getByText("store-no-desc (store-no-desc)"); + expect(option).toHaveAttribute("data-option-title", "store-no-desc"); + }); + + it("should use vector_store_id as option value", async () => { + renderComponent(); + await waitForDataFetch(); + + const option1 = screen.getByText("My Store (store-1)"); + expect(option1).toHaveAttribute("data-option-value", "store-1"); + }); + + it("should handle empty vector stores array", async () => { + mockVectorStoreListCall.mockResolvedValueOnce({ + data: [], + }); + + renderComponent(); + await waitForDataFetch(); + + const options = getOptionElements(); + expect(options.length).toBe(0); + }); + + it("should handle response without data property", async () => { + mockVectorStoreListCall.mockResolvedValueOnce({}); + + renderComponent(); + await waitForDataFetch(); + + const options = getOptionElements(); + expect(options.length).toBe(0); + }); + }); + + describe("Value prop", () => { + it("should set initial value when value prop is provided", async () => { + renderComponent({ value: ["store-1", "store-2"] }); + await waitForDataFetch(); + + const select = getSelectElement(); + const dataValue = select.getAttribute("data-value"); + expect(dataValue).toBe(JSON.stringify(["store-1", "store-2"])); + }); + + it("should handle empty value array", async () => { + renderComponent({ value: [] }); + await waitForDataFetch(); + + const select = getSelectElement(); + const dataValue = select.getAttribute("data-value"); + expect(dataValue).toBe(JSON.stringify([])); + }); + + it("should handle undefined value", async () => { + renderComponent({ value: undefined }); + await waitForDataFetch(); + + const select = getSelectElement(); + const dataValue = select.getAttribute("data-value"); + expect(dataValue).toBeNull(); // undefined value results in no data-value attribute + }); + }); + + describe("onChange callback", () => { + it("should call onChange when selection changes", async () => { + renderComponent(); + await waitForDataFetch(); + + const select = getSelectElement(); + // Simulate selecting store-1 by setting test data attribute + select.setAttribute("data-test-selection", '["store-1"]'); + fireEvent.click(select); + + expect(mockOnChange).toHaveBeenCalledWith(["store-1"]); + }); + + it("should call onChange with multiple selected values", async () => { + renderComponent(); + await waitForDataFetch(); + + const select = getSelectElement(); + // Simulate selecting multiple values + select.setAttribute("data-test-selection", '["store-1", "store-2"]'); + fireEvent.click(select); + + expect(mockOnChange).toHaveBeenCalledWith(["store-1", "store-2"]); + }); + + it("should call onChange when deselecting options", async () => { + renderComponent({ value: ["store-1", "store-2"] }); + await waitForDataFetch(); + + const select = getSelectElement(); + // Simulate deselecting store-1 + select.setAttribute("data-test-selection", '["store-2"]'); + fireEvent.click(select); + + expect(mockOnChange).toHaveBeenCalledWith(["store-2"]); + }); + }); + + describe("Error handling", () => { + it("should handle fetch errors gracefully", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + const error = new Error("Network error"); + mockVectorStoreListCall.mockRejectedValueOnce(error); + + renderComponent(); + await waitForDataFetch(); + + expect(consoleErrorSpy).toHaveBeenCalledWith("Error fetching vector stores:", error); + consoleErrorSpy.mockRestore(); + }); + + it("should not crash when fetch throws non-Error", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mockVectorStoreListCall.mockRejectedValueOnce("String error"); + + renderComponent(); + await waitForDataFetch(); + + expect(consoleErrorSpy).toHaveBeenCalledWith("Error fetching vector stores:", "String error"); + consoleErrorSpy.mockRestore(); + }); + + it("should continue to work after error", async () => { + const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + mockVectorStoreListCall.mockRejectedValueOnce(new Error("Network error")); + + renderComponent(); + await waitForDataFetch(); + + // Component should still render + expect(getSelectElement()).toBeInTheDocument(); + consoleErrorSpy.mockRestore(); + }); + }); + + describe("Edge cases", () => { + it("should handle vector stores with all optional fields missing", async () => { + const minimalStores: VectorStore[] = [ + { + vector_store_id: "minimal-store", + custom_llm_provider: "openai", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }, + ]; + mockVectorStoreListCall.mockResolvedValueOnce({ + data: minimalStores, + }); + + renderComponent(); + await waitForDataFetch(); + + expect(screen.getByText("minimal-store (minimal-store)")).toBeInTheDocument(); + const option = screen.getByText("minimal-store (minimal-store)"); + expect(option).toHaveAttribute("data-option-title", "minimal-store"); + }); + + it("should handle very long vector store names", async () => { + const longNameStores: VectorStore[] = [ + { + vector_store_id: "store-long", + custom_llm_provider: "openai", + vector_store_name: "A".repeat(200), + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }, + ]; + mockVectorStoreListCall.mockResolvedValueOnce({ + data: longNameStores, + }); + + renderComponent(); + await waitForDataFetch(); + + const expectedLabel = `${"A".repeat(200)} (store-long)`; + expect(screen.getByText(expectedLabel)).toBeInTheDocument(); + }); + + it("should handle special characters in vector store names", async () => { + const specialCharStores: VectorStore[] = [ + { + vector_store_id: "store-special", + custom_llm_provider: "openai", + vector_store_name: 'Store & Co. "Quotes"', + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }, + ]; + mockVectorStoreListCall.mockResolvedValueOnce({ + data: specialCharStores, + }); + + renderComponent(); + await waitForDataFetch(); + + expect(screen.getByText(/Store & Co\. "Quotes"/)).toBeInTheDocument(); + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx new file mode 100644 index 00000000000..16d5d3623eb --- /dev/null +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx @@ -0,0 +1,415 @@ +import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import VectorStoreTable from "./VectorStoreTable"; +import { VectorStore } from "./types"; + +// Mock dependencies +const mockGetProviderLogoAndName = vi.fn(); +const mockTableIconActionButton = vi.fn(); + +vi.mock("../provider_info_helpers", () => ({ + getProviderLogoAndName: (...args: any[]) => mockGetProviderLogoAndName(...args), +})); + +vi.mock("../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton", () => ({ + default: (props: any) => { + mockTableIconActionButton(props); + return ( + + ); + }, +})); + +// Mock Tremor components to avoid complex styling issues +vi.mock("@tremor/react", () => ({ + Table: ({ children, ...props }: any) => {children}
, + TableHead: ({ children, ...props }: any) => {children}, + TableBody: ({ children, ...props }: any) => {children}, + TableRow: ({ children, ...props }: any) => {children}, + TableHeaderCell: ({ children, ...props }: any) => {children}, + TableCell: ({ children, ...props }: any) => {children}, +})); + +// Mock antd Tooltip +vi.mock("antd", () => ({ + Tooltip: ({ children, title }: any) => ( +
+ {children} +
+ ), +})); + +// Mock Heroicons +vi.mock("@heroicons/react/outline", () => ({ + ChevronDownIcon: (props: any) =>
, + ChevronUpIcon: (props: any) =>
, + SwitchVerticalIcon: (props: any) =>
, +})); + +// Test data +const mockVectorStores: VectorStore[] = [ + { + vector_store_id: "short-id", + custom_llm_provider: "openai", + vector_store_name: "My OpenAI Store", + vector_store_description: "A store for OpenAI vectors", + created_at: "2024-01-15T10:30:00Z", + updated_at: "2024-01-15T11:00:00Z", + created_by: "user-1", + updated_by: "user-1", + }, + { + vector_store_id: "very-long-vector-store-id-that-should-be-truncated", + custom_llm_provider: "azure", + vector_store_name: undefined, // Test missing name + vector_store_description: "A store for Azure vectors with a very long description that should show a tooltip", + created_at: "2024-01-10T09:15:00Z", + updated_at: "2024-01-12T14:20:00Z", + }, + { + vector_store_id: "store-3", + custom_llm_provider: "pg_vector", + vector_store_name: "PostgreSQL Store", + vector_store_description: undefined, // Test missing description + created_at: "2024-01-05T08:00:00Z", + updated_at: "2024-01-08T16:45:00Z", + }, +]; + +// Mock functions +const mockOnView = vi.fn(); +const mockOnEdit = vi.fn(); +const mockOnDelete = vi.fn(); + +const defaultProps = { + data: mockVectorStores, + onView: mockOnView, + onEdit: mockOnEdit, + onDelete: mockOnDelete, +}; + +// Helper function to render component +const renderComponent = (props = {}) => { + return render(); +}; + +describe("VectorStoreTable", () => { + beforeEach(() => { + vi.clearAllMocks(); + + // Setup default mock returns for getProviderLogoAndName + mockGetProviderLogoAndName.mockImplementation((provider: string) => { + const providerMap: Record = { + openai: { displayName: "OpenAI", logo: "/openai-logo.png" }, + azure: { displayName: "Azure", logo: "/azure-logo.png" }, + pg_vector: { displayName: "PostgreSQL Vector", logo: "/pg-logo.png" }, + }; + return providerMap[provider] || { displayName: provider, logo: "" }; + }); + }); + + describe("Rendering", () => { + it("should render the table with data", () => { + renderComponent(); + expect(screen.getByRole("table")).toBeInTheDocument(); + }); + + it("should render table headers", () => { + renderComponent(); + expect(screen.getByText("Vector Store ID")).toBeInTheDocument(); + expect(screen.getByText("Name")).toBeInTheDocument(); + expect(screen.getByText("Description")).toBeInTheDocument(); + expect(screen.getByText("Provider")).toBeInTheDocument(); + expect(screen.getByText("Created At")).toBeInTheDocument(); + expect(screen.getByText("Updated At")).toBeInTheDocument(); + // Check that we have the expected number of header cells (6 data + 1 actions) + const headers = screen.getAllByRole("columnheader"); + expect(headers).toHaveLength(7); + }); + + it("should render all vector store rows", () => { + renderComponent(); + expect(screen.getAllByRole("row")).toHaveLength(mockVectorStores.length + 1); // +1 for header row + }); + + it("should render empty state when no data", () => { + renderComponent({ data: [] }); + expect(screen.getByText("No vector stores found")).toBeInTheDocument(); + }); + }); + + describe("Vector Store ID Column", () => { + it("should render short vector store IDs fully", () => { + renderComponent(); + expect(screen.getByText("short-id")).toBeInTheDocument(); + }); + + it("should truncate long vector store IDs", () => { + renderComponent(); + // Check that the truncated text is rendered (first 15 chars + ...) + const truncatedText = "very-long-vecto..."; + expect(screen.getByText(truncatedText)).toBeInTheDocument(); + }); + + it("should make vector store ID clickable", async () => { + const user = userEvent.setup(); + renderComponent(); + const idButton = screen.getByText("short-id"); + await user.click(idButton); + expect(mockOnView).toHaveBeenCalledWith("short-id"); + }); + + it("should have correct styling for vector store ID button", () => { + renderComponent(); + const idButton = screen.getByText("short-id").closest("button"); + expect(idButton).toHaveClass("font-mono", "text-blue-500", "bg-blue-50", "hover:bg-blue-100"); + }); + }); + + describe("Name Column", () => { + it("should render vector store name", () => { + renderComponent(); + expect(screen.getByText("My OpenAI Store")).toBeInTheDocument(); + }); + + it("should render fallback for missing name", () => { + renderComponent(); + const fallbackElements = screen.getAllByText("-"); + expect(fallbackElements.length).toBe(2); // One for missing name, one for missing description + }); + + it("should wrap name in tooltip", () => { + renderComponent(); + const tooltips = screen.getAllByTestId("tooltip"); + const nameTooltip = tooltips.find((t) => t.getAttribute("data-title") === "My OpenAI Store"); + expect(nameTooltip).toBeInTheDocument(); + }); + }); + + describe("Description Column", () => { + it("should render vector store description", () => { + renderComponent(); + expect(screen.getByText("A store for OpenAI vectors")).toBeInTheDocument(); + }); + + it("should render fallback for missing description", () => { + renderComponent(); + const fallbackElements = screen.getAllByText("-"); + expect(fallbackElements.length).toBe(2); // One for missing name, one for missing description + }); + + it("should wrap description in tooltip", () => { + renderComponent(); + const tooltips = screen.getAllByTestId("tooltip"); + const descTooltip = tooltips.find( + (t) => + t.getAttribute("data-title") === + "A store for Azure vectors with a very long description that should show a tooltip", + ); + expect(descTooltip).toBeInTheDocument(); + }); + }); + + describe("Provider Column", () => { + it("should render provider display name", () => { + renderComponent(); + expect(screen.getByText("OpenAI")).toBeInTheDocument(); + expect(screen.getByText("Azure")).toBeInTheDocument(); + expect(screen.getByText("PostgreSQL Vector")).toBeInTheDocument(); + }); + + it("should render provider logo when available", () => { + renderComponent(); + const logos = screen.getAllByRole("img"); + expect(logos).toHaveLength(3); // All providers have logos in our mock + expect(logos[0]).toHaveAttribute("src", "/openai-logo.png"); + expect(logos[0]).toHaveAttribute("alt", "OpenAI"); + }); + + it("should call getProviderLogoAndName for each provider", () => { + renderComponent(); + expect(mockGetProviderLogoAndName).toHaveBeenCalledWith("openai"); + expect(mockGetProviderLogoAndName).toHaveBeenCalledWith("azure"); + expect(mockGetProviderLogoAndName).toHaveBeenCalledWith("pg_vector"); + }); + }); + + describe("Date Columns", () => { + it("should render created at dates", () => { + renderComponent(); + const dateElements = screen.getAllByText(/1\/\d+\/2024/); + expect(dateElements.length).toBe(6); // 3 created_at + 3 updated_at dates + }); + + it("should render updated at dates", () => { + renderComponent(); + const dateElements = screen.getAllByText(/1\/\d+\/2024/); + expect(dateElements.length).toBe(6); // 3 created_at + 3 updated_at dates + }); + }); + + describe("Actions Column", () => { + it("should render edit and delete action buttons for each row", () => { + renderComponent(); + expect(screen.getAllByTestId("action-button-edit")).toHaveLength(mockVectorStores.length); + expect(screen.getAllByTestId("action-button-delete")).toHaveLength(mockVectorStores.length); + }); + + it("should call onEdit when edit button is clicked", async () => { + const user = userEvent.setup(); + renderComponent(); + const editButtons = screen.getAllByTestId("action-button-edit"); + await user.click(editButtons[0]); + expect(mockOnEdit).toHaveBeenCalledWith("short-id"); + }); + + it("should call onDelete when delete button is clicked", async () => { + const user = userEvent.setup(); + renderComponent(); + const deleteButtons = screen.getAllByTestId("action-button-delete"); + await user.click(deleteButtons[0]); + expect(mockOnDelete).toHaveBeenCalledWith("short-id"); + }); + + it("should pass correct props to TableIconActionButton", () => { + renderComponent(); + expect(mockTableIconActionButton).toHaveBeenCalledWith( + expect.objectContaining({ + variant: "Edit", + tooltipText: "Edit vector store", + onClick: expect.any(Function), + }), + ); + expect(mockTableIconActionButton).toHaveBeenCalledWith( + expect.objectContaining({ + variant: "Delete", + tooltipText: "Delete vector store", + onClick: expect.any(Function), + }), + ); + }); + }); + + describe("Sorting", () => { + it("should initialize with created_at descending sort", () => { + renderComponent(); + // The table should initialize with sorting state + expect(screen.getByTestId("chevron-down")).toBeInTheDocument(); + }); + + it("should render sort icons for sortable columns", () => { + renderComponent(); + // Should have sort icons for Created At and Updated At columns + const sortIcons = screen.getAllByTestId(/^chevron-(up|down)$|^switch-vertical$/); + expect(sortIcons.length).toBeGreaterThan(0); + }); + + it("should make header cells clickable for sorting", () => { + renderComponent(); + const headerCells = screen.getAllByRole("columnheader"); + const sortableHeaders = headerCells.filter((cell) => cell.textContent !== ""); + expect(sortableHeaders.length).toBeGreaterThan(0); + }); + + it("should show ascending icon when sorted ascending", () => { + renderComponent(); + // Initially shows descending, but we can test the logic by checking the icons are present + expect(screen.getByTestId("chevron-down")).toBeInTheDocument(); + }); + }); + + describe("Styling and Layout", () => { + it("should apply correct CSS classes to table container", () => { + renderComponent(); + const tableContainer = screen.getByRole("table").parentElement?.parentElement; + expect(tableContainer).toHaveClass("rounded-lg", "custom-border", "relative"); + }); + + it("should apply overflow styling to table wrapper", () => { + renderComponent(); + const tableWrapper = screen.getByRole("table").parentElement; + expect(tableWrapper).toHaveClass("overflow-x-auto"); + }); + + it("should apply sticky styling to actions column", () => { + renderComponent(); + const headerCells = screen.getAllByRole("columnheader"); + const actionsHeader = headerCells[headerCells.length - 1]; + expect(actionsHeader).toHaveClass("sticky", "right-0", "bg-white"); + }); + + it("should apply sticky styling to action cells", () => { + renderComponent(); + const rows = screen.getAllByRole("row").slice(1); // Skip header row + rows.forEach((row) => { + const cells = row.querySelectorAll("td"); + const lastCell = cells[cells.length - 1]; + expect(lastCell).toHaveClass("sticky", "right-0", "bg-white"); + }); + }); + }); + + describe("Table Row Styling", () => { + it("should apply correct height to table rows", () => { + renderComponent(); + const rows = screen.getAllByRole("row").slice(1); // Skip header row + rows.forEach((row) => { + expect(row).toHaveClass("h-8"); + }); + }); + + it("should apply correct cell padding and styling", () => { + renderComponent(); + const cells = screen.getAllByRole("cell"); + cells.forEach((cell) => { + expect(cell).toHaveClass("py-0.5", "max-h-8", "overflow-hidden", "text-ellipsis", "whitespace-nowrap"); + }); + }); + }); + + describe("Empty State", () => { + it("should render single row with centered message when no data", () => { + renderComponent({ data: [] }); + const rows = screen.getAllByRole("row"); + expect(rows).toHaveLength(2); // Header + empty state row + expect(screen.getByText("No vector stores found")).toBeInTheDocument(); + }); + + it("should span all columns in empty state", () => { + renderComponent({ data: [] }); + const emptyCell = screen.getByText("No vector stores found").closest("td"); + expect(emptyCell).toHaveAttribute("colSpan", "7"); // 6 data columns + 1 actions column + }); + }); + + describe("Data Edge Cases", () => { + it("should handle vector stores with minimal data", () => { + const minimalData: VectorStore[] = [ + { + vector_store_id: "minimal", + custom_llm_provider: "test", + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }, + ]; + + renderComponent({ data: minimalData }); + expect(screen.getByText("minimal")).toBeInTheDocument(); + expect(screen.getAllByText("-")).toHaveLength(2); // Name and description fallbacks + }); + + it("should handle single vector store", () => { + const singleData = [mockVectorStores[0]]; + renderComponent({ data: singleData }); + expect(screen.getAllByRole("row")).toHaveLength(2); // Header + 1 data row + }); + }); +}); diff --git a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts index 1993819be88..8b066e6a8ea 100644 --- a/ui/litellm-dashboard/src/utils/cookieUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/cookieUtils.test.ts @@ -44,6 +44,78 @@ describe("cookieUtils", () => { expect(getCookie("token")).toBeNull(); }); + + it("should return early when window is undefined (server-side rendering)", () => { + const originalWindow = global.window; + const originalDocument = global.document; + + // Mock server-side environment + delete (global as any).window; + delete (global as any).document; + + // This should not throw an error and should return early + expect(() => clearTokenCookies()).not.toThrow(); + + // Restore globals + global.window = originalWindow; + global.document = originalDocument; + }); + + it("should return early when document is undefined (server-side rendering)", () => { + const originalDocument = global.document; + + // Mock server-side environment where document is undefined + delete (global as any).document; + + // This should not throw an error and should return early + expect(() => clearTokenCookies()).not.toThrow(); + + // Restore globals + global.document = originalDocument; + }); + + it("should add current path directory to paths array when different from root and /ui", () => { + // Mock window.location.pathname using vi.stubGlobal + const originalLocation = window.location; + vi.stubGlobal('location', { ...originalLocation, pathname: '/custom/path/page.html' }); + + // Spy on document.cookie to verify the paths being used + const cookieSpy = vi.spyOn(document, 'cookie', 'set'); + + clearTokenCookies(); + + // Verify that cookies were cleared for /custom/path/ path + expect(cookieSpy).toHaveBeenCalledWith( + expect.stringContaining('path=/custom/path/') + ); + + vi.restoreAllMocks(); + }); + + it("should not add current path directory when it's already in paths array", () => { + // Mock window.location.pathname using vi.stubGlobal + const originalLocation = window.location; + vi.stubGlobal('location', { ...originalLocation, pathname: '/' }); + + // Spy on document.cookie to count calls + const cookieSpy = vi.spyOn(document, 'cookie', 'set'); + + clearTokenCookies(); + + // Count how many times each path was used + const rootPathCalls = cookieSpy.mock.calls.filter(call => + call[0].includes('path=/;') || call[0].includes('path=/ ') + ); + const uiPathCalls = cookieSpy.mock.calls.filter(call => + call[0].includes('path=/ui;') || call[0].includes('path=/ui ') + ); + + // Should have calls for root and /ui paths, but not duplicate root + expect(rootPathCalls.length).toBeGreaterThan(0); + expect(uiPathCalls.length).toBeGreaterThan(0); + + vi.restoreAllMocks(); + }); }); describe("getCookie", () => { diff --git a/ui/litellm-dashboard/src/utils/proxyUtils.test.ts b/ui/litellm-dashboard/src/utils/proxyUtils.test.ts new file mode 100644 index 00000000000..37bbd429db9 --- /dev/null +++ b/ui/litellm-dashboard/src/utils/proxyUtils.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { fetchProxySettings } from "./proxyUtils"; +import { getProxyUISettings } from "@/components/networking"; + +vi.mock("@/components/networking", () => ({ + getProxyUISettings: vi.fn(), +})); + +describe("fetchProxySettings", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("should return null when accessToken is null", async () => { + const result = await fetchProxySettings(null); + + expect(result).toBeNull(); + expect(getProxyUISettings).not.toHaveBeenCalled(); + }); + + it("should return null when accessToken is undefined", async () => { + const result = await fetchProxySettings(undefined as any); + + expect(result).toBeNull(); + expect(getProxyUISettings).not.toHaveBeenCalled(); + }); + + it("should return proxy settings when getProxyUISettings succeeds", async () => { + const mockProxySettings = { someSetting: "value", anotherSetting: 123 }; + const accessToken = "test-token"; + + vi.mocked(getProxyUISettings).mockResolvedValue(mockProxySettings); + + const result = await fetchProxySettings(accessToken); + + expect(result).toEqual(mockProxySettings); + expect(getProxyUISettings).toHaveBeenCalledOnce(); + expect(getProxyUISettings).toHaveBeenCalledWith(accessToken); + }); + + it("should return null and log error when getProxyUISettings throws", async () => { + const accessToken = "test-token"; + const mockError = new Error("Network error"); + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + vi.mocked(getProxyUISettings).mockRejectedValue(mockError); + + const result = await fetchProxySettings(accessToken); + + expect(result).toBeNull(); + expect(getProxyUISettings).toHaveBeenCalledOnce(); + expect(getProxyUISettings).toHaveBeenCalledWith(accessToken); + expect(consoleSpy).toHaveBeenCalledWith("Error fetching proxy settings:", mockError); + + consoleSpy.mockRestore(); + }); + + it("should return null and log error when getProxyUISettings throws a string", async () => { + const accessToken = "test-token"; + const mockError = "String error"; + const consoleSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + vi.mocked(getProxyUISettings).mockRejectedValue(mockError); + + const result = await fetchProxySettings(accessToken); + + expect(result).toBeNull(); + expect(getProxyUISettings).toHaveBeenCalledOnce(); + expect(getProxyUISettings).toHaveBeenCalledWith(accessToken); + expect(consoleSpy).toHaveBeenCalledWith("Error fetching proxy settings:", mockError); + + consoleSpy.mockRestore(); + }); +}); \ No newline at end of file diff --git a/ui/litellm-dashboard/src/utils/textUtils.test.ts b/ui/litellm-dashboard/src/utils/textUtils.test.ts index b7c91b9dd33..dfb37ad63b4 100644 --- a/ui/litellm-dashboard/src/utils/textUtils.test.ts +++ b/ui/litellm-dashboard/src/utils/textUtils.test.ts @@ -5,6 +5,15 @@ describe("formatLabel", () => { it("should format label", () => { expect(formatLabel("test_label")).toBe("Test Label"); }); + + it("should return empty string when text is empty string", () => { + expect(formatLabel("")).toBe(""); + }); + + it("should return the same value when text is falsy", () => { + expect(formatLabel(null as any)).toBe(null); + expect(formatLabel(undefined as any)).toBe(undefined); + }); }); describe("truncateString", () => { @@ -26,4 +35,16 @@ describe("formItemValidateJSON", () => { it("should reject with an error message for invalid JSON", async () => { await expect(formItemValidateJSON({}, "invalid JSON")).rejects.toBe("Please enter valid JSON"); }); + + it("should resolve when value is empty string", async () => { + await expect(formItemValidateJSON({}, "")).resolves.toBeUndefined(); + }); + + it("should resolve when value is null", async () => { + await expect(formItemValidateJSON({}, null as any)).resolves.toBeUndefined(); + }); + + it("should resolve when value is undefined", async () => { + await expect(formItemValidateJSON({}, undefined as any)).resolves.toBeUndefined(); + }); }); From 1184db079ec96bee585b08f5e62971534774048f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 17:30:52 -0800 Subject: [PATCH 232/388] fixing tests --- .../Modals/EditSSOSettingsModal.tsx | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx index 297698a7ba0..a731af68ff1 100644 --- a/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx +++ b/ui/litellm-dashboard/src/components/Settings/AdminSettings/SSOSettings/Modals/EditSSOSettingsModal.tsx @@ -88,17 +88,22 @@ const EditSSOSettingsModal: React.FC = ({ isVisible, // Enhanced form submission handler const handleFormSubmit = async (formValues: Record) => { - const payload = processSSOSettingsPayload(formValues); + try { + const payload = processSSOSettingsPayload(formValues); - await mutateAsync(payload, { - onSuccess: () => { - NotificationsManager.success("SSO settings updated successfully"); - onSuccess(); - }, - onError: (error) => { - NotificationsManager.fromBackend("Failed to save SSO settings: " + parseErrorMessage(error)); - }, - }); + await mutateAsync(payload, { + onSuccess: () => { + NotificationsManager.success("SSO settings updated successfully"); + onSuccess(); + }, + onError: (error) => { + NotificationsManager.fromBackend("Failed to save SSO settings: " + parseErrorMessage(error)); + }, + }); + } catch (error) { + // Handle processing errors gracefully + NotificationsManager.fromBackend("Failed to process SSO settings: " + parseErrorMessage(error)); + } }; const handleCancel = () => { From 816124a40bc8e9d0614e11be10becd5a33352d6c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 17:39:59 -0800 Subject: [PATCH 233/388] Fixign build --- .../vector_store_management/VectorStoreSelector.test.tsx | 6 ++---- .../vector_store_management/VectorStoreTable.test.tsx | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx index 8c6b85a53de..67476b5559d 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreSelector.test.tsx @@ -1,9 +1,7 @@ -import { render, screen, waitFor, fireEvent } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import VectorStoreSelector from "./VectorStoreSelector"; -import { vectorStoreListCall } from "../networking"; import { VectorStore } from "./types"; +import VectorStoreSelector from "./VectorStoreSelector"; // Mock dependencies const mockVectorStoreListCall = vi.fn(); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx index 16d5d3623eb..65d15260c4c 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import { render, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; import VectorStoreTable from "./VectorStoreTable"; From 1112974112ecc4aacf7ca1ab811c7e3cbac48b8f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 3 Jan 2026 19:22:38 -0800 Subject: [PATCH 234/388] Virtual Keys Table Loading State --- .../VirtualKeysPage/VirtualKeysTable.test.tsx | 132 +++++++++++++++++- .../VirtualKeysPage/VirtualKeysTable.tsx | 21 +-- 2 files changed, 142 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx index 3f55b11769c..cbd3d2c7320 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.test.tsx @@ -1,4 +1,4 @@ -import { screen, waitFor } from "@testing-library/react"; +import { screen, waitFor, fireEvent } from "@testing-library/react"; import { vi, it, expect, beforeEach, MockedFunction } from "vitest"; import { renderWithProviders } from "../../../tests/test-utils"; import { VirtualKeysTable } from "./VirtualKeysTable"; @@ -264,3 +264,133 @@ it("should show skeleton loaders when isLoading is true", () => { expect(screen.queryByText("Test Key Alias")).not.toBeInTheDocument(); expect(screen.queryByText("Test Team")).not.toBeInTheDocument(); }); + +it("should show 'No keys found' message when filteredKeys is empty", () => { + // Mock empty filteredKeys + mockUseFilterLogic.mockReturnValue({ + filters: { + "Team ID": "", + "Organization ID": "", + "Key Alias": "", + "User ID": "", + "Sort By": "created_at", + "Sort Order": "desc", + }, + filteredKeys: [], + allKeyAliases: [], + allTeams: [mockTeam], + allOrganizations: [mockOrganization], + handleFilterChange: vi.fn(), + handleFilterReset: vi.fn(), + }); + + const mockProps = { + teams: [mockTeam], + organizations: [mockOrganization], + onSortChange: vi.fn(), + currentSort: { + sortBy: "created_at", + sortOrder: "desc" as const, + }, + }; + + renderWithProviders(); + + expect(screen.getByText("No keys found")).toBeInTheDocument(); +}); + +it("should handle models with more than 3 entries to trigger expansion UI", () => { + const keyWithManyModels = { + ...mockKey, + models: ["gpt-3.5-turbo", "gpt-4", "gpt-4-turbo", "claude-3", "claude-3-5-sonnet"], + }; + + mockUseFilterLogic.mockReturnValue({ + filters: { + "Team ID": "", + "Organization ID": "", + "Key Alias": "", + "User ID": "", + "Sort By": "created_at", + "Sort Order": "desc", + }, + filteredKeys: [keyWithManyModels], + allKeyAliases: ["test-key-alias"], + allTeams: [mockTeam], + allOrganizations: [mockOrganization], + handleFilterChange: vi.fn(), + handleFilterReset: vi.fn(), + }); + + const mockProps = { + teams: [mockTeam], + organizations: [mockOrganization], + onSortChange: vi.fn(), + currentSort: { + sortBy: "created_at", + sortOrder: "desc" as const, + }, + }; + + renderWithProviders(); + + // This test ensures the ChevronDownIcon import (line 6) is used + // by having a key with > 3 models which triggers the expansion logic + // that uses ChevronDownIcon and ChevronRightIcon + expect(screen.getByText("Test Key Alias")).toBeInTheDocument(); +}); + +it("should render table headers correctly", () => { + const mockProps = { + teams: [mockTeam], + organizations: [mockOrganization], + onSortChange: vi.fn(), + currentSort: { + sortBy: "created_at", + sortOrder: "desc" as const, + }, + }; + + renderWithProviders(); + + // Check that main headers are rendered (testing the header.isPlaceholder condition path) + expect(screen.getByText("Key ID")).toBeInTheDocument(); + expect(screen.getByText("Key Alias")).toBeInTheDocument(); + expect(screen.getByText("Team Alias")).toBeInTheDocument(); + expect(screen.getByText("Models")).toBeInTheDocument(); + expect(screen.getByText("Spend (USD)")).toBeInTheDocument(); +}); + +it("should handle column resizing hover events", () => { + const mockProps = { + teams: [mockTeam], + organizations: [mockOrganization], + onSortChange: vi.fn(), + currentSort: { + sortBy: "created_at", + sortOrder: "desc" as const, + }, + }; + + renderWithProviders(); + + // Find a header cell with data-header-id attribute + const headerCell = document.querySelector("[data-header-id]") as HTMLElement; + + expect(headerCell).toBeInTheDocument(); + + // Check that the resizer element exists within the header + const resizer = headerCell?.querySelector(".resizer") as HTMLElement; + expect(resizer).toBeInTheDocument(); + + // Initially, resizer should have opacity 0 + expect(resizer.style.opacity).toBe("0"); + + // Simulate mouse enter using fireEvent - should set opacity to 0.5 (lines 612-616) + fireEvent.mouseEnter(headerCell); + expect(resizer.style.opacity).toBe("0.5"); + + // Simulate mouse leave using fireEvent - should set opacity back to 0 (lines 618-622) + fireEvent.mouseLeave(headerCell); + expect(resizer.style.opacity).toBe("0"); +}); diff --git a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx index b95d675979c..3bda8ee2f02 100644 --- a/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx +++ b/ui/litellm-dashboard/src/components/VirtualKeysPage/VirtualKeysTable.tsx @@ -68,12 +68,13 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo }); const [tablePagination, setTablePagination] = React.useState({ pageIndex: 0, - pageSize: 100, + pageSize: 50, }); const { data: keys, isPending: isLoading, + isFetching, refetch, } = useKeys(tablePagination.pageIndex + 1, tablePagination.pageSize); const totalCount = keys?.total_count || 0; @@ -545,8 +546,8 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo
- {isLoading ? ( - + {isLoading || isFetching ? ( + ) : ( Showing {rangeLabel} of {totalCount} results @@ -554,32 +555,32 @@ export function VirtualKeysTable({ teams, organizations, onSortChange, currentSo )}
- {isLoading ? ( - + {isLoading || isFetching ? ( + ) : ( Page {pageIndex + 1} of {table.getPageCount()} )} - {isLoading ? ( + {isLoading || isFetching ? ( ) : ( )} - {isLoading ? ( + {isLoading || isFetching ? ( ) : ( -
- {isFetching ? ( -
- Loading configuration... -
- ) : Object.keys(discountConfig).length > 0 ? ( - - ) : ( -
- + +
+ Provider Discounts + + Apply percentage-based discounts to reduce costs for specific providers + +
+ + + + + Discounts + Test It + + + +
+
+
- )} -
- - -
- -
- - - - - + {isFetching ? ( +
+ Loading configuration... +
+ ) : Object.keys(discountConfig).length > 0 ? ( + + ) : ( +
+ + + + + No provider discounts configured + + + Click "Add Provider Discount" to get started + +
+ )} +
+ + +
+ +
+ + + + + + )} - {/* Accordion 2: Fee/Price Margin */} - + {/* Accordion 2: Fee/Price Margin - Only for proxy admins */} + {isProxyAdmin && ( + + +
+ Fee/Price Margin + + Add fees or margins to LLM costs for internal billing and cost recovery + +
+ + +
+
+ +
+ {isFetching ? ( +
+ Loading configuration... +
+ ) : Object.keys(marginConfig).length > 0 ? ( + + ) : ( +
+ + + + + No provider margins configured + + + Click "Add Provider Margin" to get started + +
+ )} +
+ + + )} + + {/* Accordion 3: Pricing Calculator - Available to all roles */} +
- Fee/Price Margin + Pricing Calculator - Add fees or margins to LLM costs for internal billing and cost recovery + Estimate LLM costs based on expected token usage and request volume
-
- -
- {isFetching ? ( -
- Loading configuration... -
- ) : Object.keys(marginConfig).length > 0 ? ( - - ) : ( -
- - - - - No provider margins configured - - - Click "Add Provider Margin" to get started - -
- )} +
diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx new file mode 100644 index 00000000000..03d5ca0e518 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/cost_results.tsx @@ -0,0 +1,203 @@ +import React from "react"; +import { Text } from "@tremor/react"; +import { Card, Statistic, Row, Col, Divider, Spin } from "antd"; +import { DollarOutlined, LoadingOutlined } from "@ant-design/icons"; +import { CostEstimateResponse } from "../types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import ExportDropdown from "./export_dropdown"; + +interface CostResultsProps { + result: CostEstimateResponse | null; + loading: boolean; +} + +const formatCost = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + if (value === 0) return "$0"; + if (value < 0.0001) return `$${value.toExponential(2)}`; + if (value < 1) return `$${value.toFixed(4)}`; + return `$${formatNumberWithCommas(value, 2, true)}`; +}; + +const formatRequests = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + return formatNumberWithCommas(value, 0, true); +}; + +const CostResults: React.FC = ({ result, loading }) => { + if (!result && !loading) { + return ( +
+ + Select a model to see cost estimates + +
+ ); + } + + if (loading && !result) { + return ( +
+ } /> + Calculating costs... +
+ ); + } + + if (!result) return null; + + return ( +
+ + +
+
+ Cost Estimate + + Model: {result.model} {result.provider && `(${result.provider})`} + +
+
+ {loading && } size="small" />} + +
+
+ + + + + } + /> + + + + + + + + + 0 ? "#faad14" : undefined, + }} + /> + + + + + {result.daily_cost !== null && ( + + + + } + /> + + + + + + + + + 0 ? "#faad14" : undefined, + }} + /> + + + + )} + + {result.monthly_cost !== null && ( + + + + } + /> + + + + + + + + + 0 ? "#faad14" : undefined, + }} + /> + + + + )} + + {(result.input_cost_per_token || result.output_cost_per_token) && ( +
+ Token Pricing: + {result.input_cost_per_token && ( + Input: ${formatNumberWithCommas(result.input_cost_per_token * 1_000_000, 2)}/1M tokens + )} + {result.input_cost_per_token && result.output_cost_per_token && " | "} + {result.output_cost_per_token && ( + Output: ${formatNumberWithCommas(result.output_cost_per_token * 1_000_000, 2)}/1M tokens + )} +
+ )} +
+ ); +}; + +export default CostResults; diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx new file mode 100644 index 00000000000..e8a681021d6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_dropdown.tsx @@ -0,0 +1,71 @@ +import React, { useState, useRef, useEffect } from "react"; +import { Button } from "@tremor/react"; +import { DownloadOutlined, FilePdfOutlined, FileExcelOutlined } from "@ant-design/icons"; +import { CostEstimateResponse } from "../types"; +import { exportToPDF, exportToCSV } from "./export_utils"; + +interface ExportDropdownProps { + result: CostEstimateResponse; +} + +const ExportDropdown: React.FC = ({ result }) => { + const [isOpen, setIsOpen] = useState(false); + const menuRef = useRef(null); + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (menuRef.current && !menuRef.current.contains(event.target as Node)) { + setIsOpen(false); + } + }; + + if (isOpen) { + document.addEventListener("mousedown", handleClickOutside); + } + + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [isOpen]); + + return ( +
+ + + {isOpen && ( +
+ + +
+ )} +
+ ); +}; + +export default ExportDropdown; + diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts new file mode 100644 index 00000000000..e02e8288456 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/export_utils.ts @@ -0,0 +1,276 @@ +import { CostEstimateResponse } from "../types"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; + +const formatCostForExport = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + if (value === 0) return "$0.00"; + if (value < 0.01) return `$${value.toFixed(6)}`; + if (value < 1) return `$${value.toFixed(4)}`; + return `$${formatNumberWithCommas(value, 2)}`; +}; + +const formatRequestsForExport = (value: number | null | undefined): string => { + if (value === null || value === undefined) return "-"; + return formatNumberWithCommas(value, 0); +}; + +export const exportToPDF = (result: CostEstimateResponse): void => { + const printWindow = window.open("", "_blank"); + if (!printWindow) { + alert("Please allow popups to export PDF"); + return; + } + + const html = ` + + + + Cost Estimate Report - ${result.model} + + + +

LLM Cost Estimate Report

+ +
+

Model: ${result.model}

+ ${result.provider ? `

Provider: ${result.provider}

` : ""} +

Input Tokens per Request: ${formatRequestsForExport(result.input_tokens)}

+

Output Tokens per Request: ${formatRequestsForExport(result.output_tokens)}

+ ${result.num_requests_per_day ? `

Requests per Day: ${formatRequestsForExport(result.num_requests_per_day)}

` : ""} + ${result.num_requests_per_month ? `

Requests per Month: ${formatRequestsForExport(result.num_requests_per_month)}

` : ""} +
+ +

Per-Request Cost Breakdown

+ + + + + + + + + + + + + + + + + + + + + +
Cost TypeAmount
Input Cost${formatCostForExport(result.input_cost_per_request)}
Output Cost${formatCostForExport(result.output_cost_per_request)}
Margin/Fee${formatCostForExport(result.margin_cost_per_request)}
Total per Request${formatCostForExport(result.cost_per_request)}
+ + ${result.daily_cost !== null ? ` +

Daily Cost Estimate (${formatRequestsForExport(result.num_requests_per_day)} requests/day)

+ + + + + + + + + + + + + + + + + + + + + +
Cost TypeAmount
Input Cost${formatCostForExport(result.daily_input_cost)}
Output Cost${formatCostForExport(result.daily_output_cost)}
Margin/Fee${formatCostForExport(result.daily_margin_cost)}
Total Daily${formatCostForExport(result.daily_cost)}
+ ` : ""} + + ${result.monthly_cost !== null ? ` +

Monthly Cost Estimate (${formatRequestsForExport(result.num_requests_per_month)} requests/month)

+ + + + + + + + + + + + + + + + + + + + + +
Cost TypeAmount
Input Cost${formatCostForExport(result.monthly_input_cost)}
Output Cost${formatCostForExport(result.monthly_output_cost)}
Margin/Fee${formatCostForExport(result.monthly_margin_cost)}
Total Monthly${formatCostForExport(result.monthly_cost)}
+ ` : ""} + + ${result.input_cost_per_token || result.output_cost_per_token ? ` +

Token Pricing

+ + + + + + ${result.input_cost_per_token ? ` + + + + + ` : ""} + ${result.output_cost_per_token ? ` + + + + + ` : ""} +
Token TypePrice per 1M Tokens
Input Tokens$${(result.input_cost_per_token * 1000000).toFixed(2)}
Output Tokens$${(result.output_cost_per_token * 1000000).toFixed(2)}
+ ` : ""} + +
+

Generated by LiteLLM Pricing Calculator on ${new Date().toLocaleString()}

+
+ + + `; + + printWindow.document.write(html); + printWindow.document.close(); + printWindow.onload = () => { + printWindow.print(); + }; +}; + +export const exportToCSV = (result: CostEstimateResponse): void => { + const rows = [ + ["LLM Cost Estimate Report"], + [""], + ["Configuration"], + ["Model", result.model], + ["Provider", result.provider || "-"], + ["Input Tokens per Request", result.input_tokens.toString()], + ["Output Tokens per Request", result.output_tokens.toString()], + ["Requests per Day", result.num_requests_per_day?.toString() || "-"], + ["Requests per Month", result.num_requests_per_month?.toString() || "-"], + [""], + ["Per-Request Costs"], + ["Input Cost", result.input_cost_per_request.toString()], + ["Output Cost", result.output_cost_per_request.toString()], + ["Margin/Fee", result.margin_cost_per_request.toString()], + ["Total per Request", result.cost_per_request.toString()], + ]; + + if (result.daily_cost !== null) { + rows.push( + [""], + ["Daily Costs"], + ["Daily Input Cost", result.daily_input_cost?.toString() || "-"], + ["Daily Output Cost", result.daily_output_cost?.toString() || "-"], + ["Daily Margin/Fee", result.daily_margin_cost?.toString() || "-"], + ["Total Daily", result.daily_cost.toString()] + ); + } + + if (result.monthly_cost !== null) { + rows.push( + [""], + ["Monthly Costs"], + ["Monthly Input Cost", result.monthly_input_cost?.toString() || "-"], + ["Monthly Output Cost", result.monthly_output_cost?.toString() || "-"], + ["Monthly Margin/Fee", result.monthly_margin_cost?.toString() || "-"], + ["Total Monthly", result.monthly_cost.toString()] + ); + } + + if (result.input_cost_per_token || result.output_cost_per_token) { + rows.push( + [""], + ["Token Pricing (per 1M tokens)"], + ["Input Token Price", result.input_cost_per_token ? `$${(result.input_cost_per_token * 1000000).toFixed(2)}` : "-"], + ["Output Token Price", result.output_cost_per_token ? `$${(result.output_cost_per_token * 1000000).toFixed(2)}` : "-"] + ); + } + + const csv = rows.map(row => row.join(",")).join("\n"); + const blob = new Blob([csv], { type: "text/csv;charset=utf-8;" }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `cost_estimate_${result.model.replace(/\//g, "_")}_${new Date().toISOString().split("T")[0]}.csv`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); +}; + diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx new file mode 100644 index 00000000000..fff6475e8a2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/index.tsx @@ -0,0 +1,31 @@ +import React, { useCallback } from "react"; +import PricingForm from "./pricing_form"; +import CostResults from "./cost_results"; +import { useCostEstimate } from "./use_cost_estimate"; +import { PricingCalculatorProps, PricingFormValues } from "./types"; + +const PricingCalculator: React.FC = ({ + accessToken, + models, +}) => { + const { loading, result, debouncedFetch } = useCostEstimate(accessToken); + + const handleValuesChange = useCallback( + (_changedValues: Partial, allValues: PricingFormValues) => { + if (allValues.model) { + debouncedFetch(allValues); + } + }, + [debouncedFetch] + ); + + return ( +
+ + +
+ ); +}; + +export default PricingCalculator; + diff --git a/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx new file mode 100644 index 00000000000..c91a19e5516 --- /dev/null +++ b/ui/litellm-dashboard/src/components/CostTrackingSettings/pricing_calculator/pricing_form.tsx @@ -0,0 +1,104 @@ +import React from "react"; +import { Form, InputNumber, Select, Row, Col } from "antd"; +import { PricingFormValues } from "./types"; + +interface PricingFormProps { + models: string[]; + onValuesChange: (changedValues: Partial, allValues: PricingFormValues) => void; +} + +const PricingForm: React.FC = ({ models, onValuesChange }) => { + return ( +
+ + + + handleEntryChange(record.id, "model", value)} + optionFilterProp="label" + filterOption={(input, option) => + String(option?.label ?? "").toLowerCase().includes(input.toLowerCase()) + } + options={models.map((model) => ({ + value: model, + label: model, + }))} + style={{ width: "100%" }} + size="small" + /> + ), + }, + { + title: "Input Tokens", + dataIndex: "input_tokens", + key: "input_tokens", + width: "18%", + render: (_: number, record: ModelEntry) => ( + handleEntryChange(record.id, "input_tokens", value ?? 0)} + style={{ width: "100%" }} + size="small" + formatter={(value) => `${value}`.replace(/\B(?=(\d{3})+(?!\d))/g, ",")} + /> + ), + }, + { + title: "Output Tokens", + dataIndex: "output_tokens", + key: "output_tokens", + width: "18%", + render: (_: number, record: ModelEntry) => ( + handleEntryChange(record.id, "output_tokens", value ?? 0)} + style={{ width: "100%" }} + size="small" + formatter={(value) => `${value}`.replace(/\B(?=(\d{3})+(?!\d))/g, ",")} + /> + ), + }, + { + title: `Requests/${timePeriod === "day" ? "Day" : "Month"}`, + dataIndex: timePeriod === "day" ? "num_requests_per_day" : "num_requests_per_month", + key: "num_requests", + width: "20%", + render: (_: number | undefined, record: ModelEntry) => ( + + handleEntryChange( + record.id, + timePeriod === "day" ? "num_requests_per_day" : "num_requests_per_month", + value ?? undefined + ) + } + style={{ width: "100%" }} + size="small" + placeholder="-" + formatter={(value) => (value ? `${value}`.replace(/\B(?=(\d{3})+(?!\d))/g, ",") : "")} + /> + ), + }, + { + title: "", + key: "actions", + width: 50, + render: (_: unknown, record: ModelEntry) => ( +