From 9274860aa28c5d69d696077b92283dfcad90e0a5 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 14:54:26 -0800 Subject: [PATCH 01/87] Base commit --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..5094fc5de97 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -594,7 +594,7 @@ def _build_sso_user_update_data( user_id: Optional[str], ) -> dict: """ - Build the update data dictionary for SSO user upsert. + Build the update data dictionary for SSO user upsert Args: result: The SSO response containing user information From 42d7d757a3bb1e5da02ed53c494ce6abca2f7e61 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 15:48:26 -0800 Subject: [PATCH 02/87] Adding role mappings to SSOConfig DB --- litellm/proxy/management_endpoints/ui_sso.py | 2 +- litellm/proxy/proxy_server.py | 1 + .../proxy_setting_endpoints.py | 12 ++ .../proxy/management_endpoints/ui_sso.py | 34 ++++- .../test_proxy_setting_endpoints.py | 135 ++++++++++++++++++ 5 files changed, 182 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 5094fc5de97..d1db21a2706 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -594,7 +594,7 @@ def _build_sso_user_update_data( user_id: Optional[str], ) -> dict: """ - Build the update data dictionary for SSO user upsert + Build the update data dictionary for SSO user upsert. Args: result: The SSO response containing user information diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8fdea95d7ad..dfadab1d531 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3578,6 +3578,7 @@ class ProxyConfig: ) if sso_settings is not None: # Capitalize all keys in sso_settings dictionary + sso_settings.sso_settings.pop("role_mappings", None) uppercase_sso_settings = { key.upper(): value for key, value in sso_settings.sso_settings.items() diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index 9c99b625e9f..d9a41d38b22 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -433,10 +433,21 @@ async def get_sso_settings(): if sso_db_record and sso_db_record.sso_settings: # Load settings from database sso_settings_dict = dict(sso_db_record.sso_settings) + + # Extract role_mappings before removing it (it's a dict, not an env variable) + role_mappings_data = sso_settings_dict.pop("role_mappings", None) + role_mappings = None + if role_mappings_data: + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings + if isinstance(role_mappings_data, dict): + role_mappings = RoleMappings(**role_mappings_data) + elif isinstance(role_mappings_data, RoleMappings): + role_mappings = role_mappings_data decrypted_sso_settings_dict = proxy_config._decrypt_and_set_db_env_variables(environment_variables=sso_settings_dict) # Build SSO config with database values or environment fallback + sso_config = SSOConfig( google_client_id=decrypted_sso_settings_dict.get("google_client_id", None), google_client_secret=decrypted_sso_settings_dict.get("google_client_secret", None), @@ -451,6 +462,7 @@ async def get_sso_settings(): proxy_base_url=decrypted_sso_settings_dict.get("proxy_base_url", None), user_email=decrypted_sso_settings_dict.get("user_email"), ui_access_mode=decrypted_sso_settings_dict.get("ui_access_mode"), + role_mappings=role_mappings, ) # Get the schema for UI display diff --git a/litellm/types/proxy/management_endpoints/ui_sso.py b/litellm/types/proxy/management_endpoints/ui_sso.py index 820b0164400..187d8c97c05 100644 --- a/litellm/types/proxy/management_endpoints/ui_sso.py +++ b/litellm/types/proxy/management_endpoints/ui_sso.py @@ -1,10 +1,12 @@ -from typing import List, Literal, Optional, Union +from typing import Dict, List, Literal, Optional, Union from pydantic import Field from typing_extensions import TypedDict from litellm.types.utils import LiteLLMPydanticObjectBase +from litellm.proxy._types import LitellmUserRoles + class LiteLLM_UpperboundKeyGenerateParams(LiteLLMPydanticObjectBase): """ @@ -60,6 +62,30 @@ class AccessControl_UI_AccessMode(LiteLLMPydanticObjectBase): sso_group_jwt_field: str +class RoleMappings(LiteLLMPydanticObjectBase): + """ + Configuration for mapping SSO groups to LiteLLM roles. + + The system will look at the group_claim field in the SSO token to determine + which role to assign the user based on the roles mapping. + """ + + provider: str = Field( + description="SSO Provider name (e.g., 'google', 'microsoft', 'generic')" + ) + group_claim: str = Field( + description="The field name in the SSO token that contains the groups array (e.g., 'groups', 'roles')" + ) + default_role: Optional[LitellmUserRoles] = Field( + default=None, + description="Default role to assign if user's groups don't match any role mappings. Must be a valid LitellmUserRoles value (e.g., 'proxy_admin', 'internal_user', 'proxy_admin_viewer')" + ) + roles: Dict[LitellmUserRoles, List[str]] = Field( + default_factory=dict, + description="Mapping of LiteLLM role names to arrays of SSO group names. Example: {'proxy_admin': ['group-1', 'group-2'], 'proxy_admin_viewer': ['group-3']}" + ) + + class SSOConfig(LiteLLMPydanticObjectBase): """ Configuration for SSO environment variables and settings @@ -127,6 +153,12 @@ class SSOConfig(LiteLLMPydanticObjectBase): description="Access mode for the UI", ) + # Role Mappings + role_mappings: Optional[RoleMappings] = Field( + default=None, + description="Configuration for mapping SSO groups to LiteLLM roles based on group claims in the SSO token", + ) + class DefaultTeamSSOParams(LiteLLMPydanticObjectBase): """ diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index d3c99151195..8fdfd6897a8 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -290,6 +290,10 @@ class TestProxySettingEndpoints: assert "google_client_id" in data["field_schema"]["properties"] assert "description" in data["field_schema"]["properties"]["google_client_id"] + # Verify role_mappings is present in response (can be None if not set) + assert "role_mappings" in values + assert values["role_mappings"] is None + # Verify find_unique was called with correct parameters mock_prisma.db.litellm_ssoconfig.find_unique.assert_called_once() call_args = mock_prisma.db.litellm_ssoconfig.find_unique.call_args @@ -863,6 +867,10 @@ class TestProxySettingEndpoints: assert values["google_client_secret"] == "decrypted_google_secret" assert values["microsoft_client_id"] == "decrypted_microsoft_id" assert values["proxy_base_url"] == "https://decrypted.example.com" + + # Verify role_mappings is present in response (can be None if not set) + assert "role_mappings" in values + assert values["role_mappings"] is None def test_update_sso_settings_to_database(self, mock_proxy_config, mock_auth, monkeypatch): """Test updating SSO settings saves to the dedicated database table""" @@ -1062,6 +1070,7 @@ class TestProxySettingEndpoints: assert values.get("google_client_id") is None assert values.get("google_client_secret") is None assert values.get("microsoft_client_id") is None + assert values.get("role_mappings") is None def test_update_sso_settings_no_database_connection(self, mock_proxy_config, mock_auth, monkeypatch): """Test updating SSO settings when database is not connected""" @@ -1088,3 +1097,129 @@ class TestProxySettingEndpoints: data = response.json() assert "error" in data["detail"] assert "Database not connected" in data["detail"]["error"] + + def test_get_sso_settings_with_role_mappings(self, mock_proxy_config, mock_auth, monkeypatch): + """Test getting SSO settings when role_mappings is present in database""" + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy._types import LitellmUserRoles + + # Mock the prisma client with database record containing role_mappings + mock_prisma = MagicMock() + mock_db_record = MagicMock() + mock_db_record.sso_settings = { + "google_client_id": "test_google_client_id", + "role_mappings": { + "provider": "google", + "group_claim": "groups", + "default_role": LitellmUserRoles.INTERNAL_USER, + "roles": { + LitellmUserRoles.PROXY_ADMIN: ["admin-group"], + }, + }, + } + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + # Mock decryption to return the values as-is (role_mappings should not be passed to decryption) + from litellm.proxy.proxy_server import proxy_config + def mock_decrypt(environment_variables): + # role_mappings should not be in environment_variables since it's extracted before decryption + assert "role_mappings" not in environment_variables + return environment_variables + + monkeypatch.setattr( + proxy_config, "_decrypt_and_set_db_env_variables", mock_decrypt + ) + + response = client.get("/get/sso_settings") + + assert response.status_code == 200 + data = response.json() + + # Verify role_mappings is returned correctly + values = data["values"] + assert "role_mappings" in values + assert values["role_mappings"] is not None + assert values["role_mappings"]["provider"] == "google" + assert values["role_mappings"]["group_claim"] == "groups" + assert values["role_mappings"]["default_role"] == LitellmUserRoles.INTERNAL_USER + assert values["role_mappings"]["roles"][LitellmUserRoles.PROXY_ADMIN] == ["admin-group"] + + def test_role_mappings_stored_and_retrieved(self, mock_proxy_config, mock_auth, monkeypatch): + """Test that role_mappings is properly stored and retrieved from SSO settings""" + import json + from unittest.mock import AsyncMock, MagicMock + from litellm.proxy._types import LitellmUserRoles + + monkeypatch.setenv("LITELLM_SALT_KEY", "test_salt_key") + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", True) + + # Mock the prisma client + mock_prisma = MagicMock() + mock_prisma.db.litellm_ssoconfig.upsert = AsyncMock() + mock_prisma.db.litellm_config = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None) + mock_prisma.db.litellm_config.update = AsyncMock() + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + + # Mock encryption to return values as-is + from litellm.proxy.proxy_server import proxy_config + monkeypatch.setattr(proxy_config, "_encrypt_env_variables", lambda environment_variables: environment_variables) + + # SSO settings with role_mappings + role_mappings_data = { + "provider": "google", + "group_claim": "groups", + "default_role": LitellmUserRoles.INTERNAL_USER, + "roles": { + LitellmUserRoles.PROXY_ADMIN: ["admin-group"], + LitellmUserRoles.INTERNAL_USER: ["user-group"], + }, + } + + new_sso_settings = { + "google_client_id": "test_google_id", + "role_mappings": role_mappings_data, + } + + response = client.patch("/update/sso_settings", json=new_sso_settings) + + assert response.status_code == 200 + data = response.json() + assert data["status"] == "success" + assert "role_mappings" in data["settings"] + + # Verify role_mappings structure in response + returned_role_mappings = data["settings"]["role_mappings"] + assert returned_role_mappings["provider"] == "google" + assert returned_role_mappings["group_claim"] == "groups" + assert returned_role_mappings["default_role"] == LitellmUserRoles.INTERNAL_USER + assert returned_role_mappings["roles"][LitellmUserRoles.PROXY_ADMIN] == ["admin-group"] + + # Verify upsert was called with role_mappings in the data + assert mock_prisma.db.litellm_ssoconfig.upsert.called + call_args = mock_prisma.db.litellm_ssoconfig.upsert.call_args + create_data = call_args.kwargs["data"]["create"] + stored_sso_settings = json.loads(create_data["sso_settings"]) + assert "role_mappings" in stored_sso_settings + assert stored_sso_settings["role_mappings"]["provider"] == "google" + + # Now test retrieving role_mappings + mock_db_record = MagicMock() + mock_db_record.sso_settings = stored_sso_settings + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock(return_value=mock_db_record) + monkeypatch.setattr( + proxy_config, "_decrypt_and_set_db_env_variables", lambda environment_variables: environment_variables + ) + + get_response = client.get("/get/sso_settings") + assert get_response.status_code == 200 + get_data = get_response.json() + + # Verify role_mappings is returned correctly + assert "role_mappings" in get_data["values"] + retrieved_role_mappings = get_data["values"]["role_mappings"] + assert retrieved_role_mappings is not None + assert retrieved_role_mappings["provider"] == "google" + assert retrieved_role_mappings["group_claim"] == "groups" + assert retrieved_role_mappings["default_role"] == LitellmUserRoles.INTERNAL_USER From 58330f852d82bd42f02f2e920cdb14ae0b927024 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 16 Dec 2025 17:36:08 -0800 Subject: [PATCH 03/87] WIP waiting for okta --- litellm/proxy/management_endpoints/ui_sso.py | 134 ++++++++++++++++++- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index d1db21a2706..7a404bcda81 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -85,6 +85,58 @@ else: router = APIRouter() +def determine_role_from_groups( + user_groups: List[str], + role_mappings: "RoleMappings", +) -> Optional[LitellmUserRoles]: + """ + Determine the highest privilege role for a user based on their groups. + + Role hierarchy (highest to lowest): + - proxy_admin + - proxy_admin_viewer + - internal_user + - internal_user_viewer + + Args: + user_groups: List of group names from the SSO token + role_mappings: RoleMappings configuration object + + Returns: + The highest privilege role found, or default_role if no matches, or None + """ + if not role_mappings.roles: + # No role mappings configured, return default_role + return role_mappings.default_role + + # Role hierarchy (highest to lowest) + role_hierarchy = [ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + + # Convert user_groups to a set for efficient lookup + user_groups_set = set(user_groups) if isinstance(user_groups, list) else set() + + # Find the highest privilege role the user belongs to + for role in role_hierarchy: + if role in role_mappings.roles: + role_groups = role_mappings.roles[role] + if isinstance(role_groups, list) and user_groups_set.intersection(set(role_groups)): + verbose_proxy_logger.debug( + f"User groups {user_groups} matched role '{role.value}' via groups: {role_groups}" + ) + return role + + # No matching groups found, return default_role + verbose_proxy_logger.debug( + f"User groups {user_groups} did not match any role mappings, using default_role: {role_mappings.default_role}" + ) + return role_mappings.default_role + + def process_sso_jwt_access_token( access_token_str: Optional[str], sso_jwt_handler: Optional[JWTHandler], @@ -243,6 +295,7 @@ def generic_response_convertor( response, jwt_handler: JWTHandler, sso_jwt_handler: Optional[JWTHandler] = None, + role_mappings: Optional["RoleMappings"] = None, ) -> CustomOpenID: generic_user_id_attribute_name = os.getenv( "GENERIC_USER_ID_ATTRIBUTE", "preferred_username" @@ -281,16 +334,48 @@ def generic_response_convertor( team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response)) all_teams.extend(team_ids) - # Extract user role from SSO response - user_role_from_sso = get_nested_value(response, generic_user_role_attribute_name) + # Determine user role based on role_mappings if available + # Only apply role_mappings for GENERIC SSO provider user_role: Optional[LitellmUserRoles] = None - if user_role_from_sso is not None: - role = get_litellm_user_role(user_role_from_sso) - if role is not None: - user_role = role + + if role_mappings is not None and role_mappings.provider.lower() in ["generic", "okta"]: + # Use role_mappings to determine role from groups + group_claim = role_mappings.group_claim + user_groups_raw = get_nested_value(response, group_claim) + + # Handle different formats: could be a list, string (comma-separated), or single value + user_groups: List[str] = [] + if isinstance(user_groups_raw, list): + user_groups = [str(g) for g in user_groups_raw] + elif isinstance(user_groups_raw, str): + # Handle comma-separated string + user_groups = [g.strip() for g in user_groups_raw.split(",") if g.strip()] + elif user_groups_raw is not None: + # Single value + user_groups = [str(user_groups_raw)] + + if user_groups: + user_role = determine_role_from_groups(user_groups, role_mappings) verbose_proxy_logger.debug( - f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + f"Determined role '{user_role.value if user_role else None}' from groups '{user_groups}' using role_mappings" ) + else: + # No groups found, use default_role + user_role = role_mappings.default_role + verbose_proxy_logger.debug( + f"No groups found in '{group_claim}', using default_role: {role_mappings.default_role}" + ) + + # Fallback to existing logic if role_mappings not used + if user_role is None: + user_role_from_sso = get_nested_value(response, generic_user_role_attribute_name) + if user_role_from_sso is not None: + role = get_litellm_user_role(user_role_from_sso) + if role is not None: + user_role = role + verbose_proxy_logger.debug( + f"Found valid LitellmUserRoles '{role.value}' from SSO attribute '{generic_user_role_attribute_name}'" + ) return CustomOpenID( id=get_nested_value(response, generic_user_id_attribute_name), @@ -369,6 +454,40 @@ async def get_generic_sso_response( userinfo_endpoint=generic_userinfo_endpoint, ) + # Get role_mappings from SSO settings if available + role_mappings: Optional["RoleMappings"] = None + try: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Prisma client is None, connect a database to your proxy" + ) + + # Get SSO config from dedicated table + sso_db_record = await prisma_client.db.litellm_ssoconfig.find_unique( + where={"id": "sso_config"} + ) + + if sso_db_record and sso_db_record.sso_settings: + sso_settings_dict = dict(sso_db_record.sso_settings) + role_mappings_data = sso_settings_dict.get("role_mappings") + + if role_mappings_data: + from litellm.types.proxy.management_endpoints.ui_sso import RoleMappings + if isinstance(role_mappings_data, dict): + role_mappings = RoleMappings(**role_mappings_data) + elif isinstance(role_mappings_data, RoleMappings): + role_mappings = role_mappings_data + + verbose_proxy_logger.debug( + f"Loaded role_mappings for provider '{role_mappings.provider}'" + ) + except Exception as e: + # If we can't load role_mappings, continue with existing logic + verbose_proxy_logger.debug( + f"Could not load role_mappings from database: {e}. Continuing with existing role logic." + ) + def response_convertor(response, client): nonlocal received_response # return for user debugging received_response = response @@ -376,6 +495,7 @@ async def get_generic_sso_response( response=response, jwt_handler=jwt_handler, sso_jwt_handler=sso_jwt_handler, + role_mappings=role_mappings, ) SSOProvider = create_provider( From bf76e66d2ccaace63eb6353ab92e37b57d99b349 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 09:42:59 -0800 Subject: [PATCH 04/87] Working SSO Mapping new user, overrides default user settings --- litellm/proxy/management_endpoints/ui_sso.py | 48 +++++++++++++++++++- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 7a404bcda81..5afccc6fe5f 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -1173,8 +1173,44 @@ async def insert_sso_user( if user_defined_values is None: raise ValueError("user_defined_values is None") + # Check if role_mappings is configured in SSO settings + role_mappings_configured = False + try: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Prisma client is None, connect a database to your proxy" + ) + + # Get SSO config from dedicated table + sso_db_record = await prisma_client.db.litellm_ssoconfig.find_unique( + where={"id": "sso_config"} + ) + + if sso_db_record and sso_db_record.sso_settings: + sso_settings_dict = dict(sso_db_record.sso_settings) + role_mappings_data = sso_settings_dict.get("role_mappings") + role_mappings_configured = role_mappings_data is not None + except Exception as e: + # If we can't check role_mappings, continue with existing logic + verbose_proxy_logger.debug( + f"Could not check role_mappings configuration: {e}. Using default behavior." + ) + + # Apply default_internal_user_params if litellm.default_internal_user_params: - user_defined_values.update(litellm.default_internal_user_params) # type: ignore + # If role_mappings is configured and user_role is already set from SSO, preserve it + if role_mappings_configured and user_defined_values.get("user_role") is not None: + # Preserve the SSO-extracted role, but apply other defaults + preserved_role = user_defined_values.get("user_role") + user_defined_values.update(litellm.default_internal_user_params) # type: ignore + user_defined_values["user_role"] = preserved_role # Restore preserved role + verbose_proxy_logger.debug( + f"Preserved SSO-extracted role '{preserved_role}' (role_mappings configured)" + ) + else: + # Default behavior: update all values including role + user_defined_values.update(litellm.default_internal_user_params) # type: ignore # Set budget for internal users if user_defined_values.get("user_role") == LitellmUserRoles.INTERNAL_USER.value: @@ -1812,7 +1848,15 @@ class SSOAuthenticationHandler: ) user_id = getattr(result, "id", None) user_email = getattr(result, "email", None) - user_role = getattr(result, generic_user_role_attribute_name, None) # type: ignore + if user_role is None: + _role_from_attr = getattr(result, generic_user_role_attribute_name, None) # type: ignore + if _role_from_attr is not None: + # Convert enum to string if needed + user_role = ( + _role_from_attr.value + if isinstance(_role_from_attr, LitellmUserRoles) + else _role_from_attr + ) if user_id is None and result is not None: _first_name = getattr(result, "first_name", "") or "" From 313a613a13e2756308ce828924864815ac36bd38 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 10:56:11 -0800 Subject: [PATCH 05/87] Adding tests --- .../proxy/management_endpoints/test_ui_sso.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py index 500fc67de89..20829466570 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py +++ b/tests/test_litellm/proxy/management_endpoints/test_ui_sso.py @@ -3043,3 +3043,108 @@ class TestAddMissingTeamMember: assert set(added_teams) == set( expected_teams_added ), f"Expected teams {expected_teams_added}, but got {added_teams}" + + +@pytest.mark.asyncio +async def test_role_mappings_override_default_internal_user_params(): + """ + Test that when role_mappings is configured in SSO settings, + the SSO-extracted role overrides default_internal_user_params role. + """ + from litellm.proxy._types import NewUserResponse, SSOUserDefinedValues + from litellm.proxy.management_endpoints.ui_sso import insert_sso_user + + # Save original default_internal_user_params + original_default_params = getattr(litellm, "default_internal_user_params", None) + + try: + # Set default_internal_user_params with a role that should be overridden + litellm.default_internal_user_params = { + "user_role": "internal_user", + "max_budget": 100, + "budget_duration": "30d", + "models": ["gpt-3.5-turbo"], + } + + # Mock SSO result + mock_result_openid = CustomOpenID( + id="test-user-123", + email="test@example.com", + display_name="Test User", + provider="microsoft", + team_ids=[], + ) + + # User defined values with SSO-extracted role (from role_mappings) + user_defined_values: SSOUserDefinedValues = { + "user_id": "test-user-123", + "user_email": "test@example.com", + "user_role": "proxy_admin", # Role from SSO role_mappings + "max_budget": None, + "budget_duration": None, + "models": [], + } + + # Mock Prisma client with SSO config that has role_mappings configured + mock_prisma = MagicMock() + mock_sso_config = MagicMock() + mock_sso_config.sso_settings = { + "role_mappings": { + "Admin": "proxy_admin", + "User": "internal_user", + } + } + mock_prisma.db.litellm_ssoconfig.find_unique = AsyncMock( + return_value=mock_sso_config + ) + + # Mock new_user function + mock_new_user_response = NewUserResponse( + user_id="test-user-123", + key="sk-xxxxx", + teams=None, + ) + + with patch( + "litellm.proxy.utils.get_prisma_client_or_throw", + return_value=mock_prisma, + ), patch( + "litellm.proxy.management_endpoints.ui_sso.new_user", + return_value=mock_new_user_response, + ) as mock_new_user: + # Act + result = await insert_sso_user( + result_openid=mock_result_openid, + user_defined_values=user_defined_values, + ) + + # Assert - verify new_user was called with preserved SSO role + mock_new_user.assert_called_once() + call_args = mock_new_user.call_args + new_user_request = call_args.kwargs["data"] + + # The role from SSO should be preserved, not overridden by default_internal_user_params + assert ( + new_user_request.user_role == "proxy_admin" + ), "SSO-extracted role should override default_internal_user_params role" + + # Other default params should still be applied + assert ( + new_user_request.max_budget == 100 + ), "max_budget from default_internal_user_params should be applied" + assert ( + new_user_request.budget_duration == "30d" + ), "budget_duration from default_internal_user_params should be applied" + + # Note: models are applied via _update_internal_new_user_params inside new_user, + # not in insert_sso_user, so we verify user_defined_values was updated correctly + # by checking that the function completed successfully and other defaults were applied + # The models will be applied when new_user processes the request + + finally: + # Restore original default_internal_user_params + if original_default_params is not None: + litellm.default_internal_user_params = original_default_params + else: + if hasattr(litellm, "default_internal_user_params"): + delattr(litellm, "default_internal_user_params") From e92db2d718a78dc0443c801cb8467e6a5160c725 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 13:04:42 -0800 Subject: [PATCH 06/87] Improve e2e flow for cloudzero --- .../hooks/cloudzero/useCloudZeroSettings.ts | 70 +++++++++++++++---- .../CloudZeroCostTracking.tsx | 4 +- .../CloudZeroEmptyPlaceholder.test.tsx | 2 +- .../CloudZeroEmptyPlaceholder.tsx | 2 +- .../CloudZeroIntegrationSettings.tsx | 8 ++- .../components/CloudZeroCostTracking/types.ts | 8 +-- 6 files changed, 71 insertions(+), 23 deletions(-) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts index 5ccbe244e60..96f5ab2f944 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/cloudzero/useCloudZeroSettings.ts @@ -17,19 +17,35 @@ const getCloudZeroSettings = async (accessToken: string): Promise ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to fetch CloudZero settings"; + let errorMessage = "Failed to fetch CloudZero settings"; + try { + const errorData = await response.json(); + // Handle different error response formats + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + // If JSON parsing fails, use the status text + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } const data = await response.json(); + + // Check if settings are actually configured (all required fields are present) + if (!data || (!data.api_key_masked && !data.connection_id)) { + return null; + } + return data; }; @@ -77,9 +93,22 @@ const updateCloudZeroSettings = async (accessToken: string, params: UpdateParams }); if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to update CloudZero settings"; + let errorMessage = "Failed to update CloudZero settings"; + try { + const errorData = await response.json(); + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } @@ -117,9 +146,22 @@ const deleteCloudZeroSettings = async (accessToken: string): Promise ({})); - const errorMessage = - errorData?.error?.message || errorData?.message || errorData?.detail || "Failed to delete CloudZero settings"; + let errorMessage = "Failed to delete CloudZero settings"; + try { + const errorData = await response.json(); + if (typeof errorData === "object" && errorData !== null) { + errorMessage = + errorData?.error?.message || + errorData?.error || + errorData?.message || + errorData?.detail || + (typeof errorData?.error === "string" ? errorData.error : errorMessage); + } else if (typeof errorData === "string") { + errorMessage = errorData; + } + } catch { + errorMessage = response.statusText || errorMessage; + } throw new Error(errorMessage); } diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx index fbb892cb1d8..db3ea94bbf9 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroCostTracking.tsx @@ -36,7 +36,9 @@ export default function CloudZeroCostTracking() { if (error) { return ( - Error loading CloudZero settings: {error.message} + + Error loading CloudZero settings: {error instanceof Error ? error.message : String(error)} + ); } diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx index 04e0a67dea6..f7b90884006 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.test.tsx @@ -9,6 +9,6 @@ describe("CloudZeroEmptyPlaceholder", () => { expect(screen.getByText("No CloudZero Integration Found")).toBeInTheDocument(); expect(screen.getByText(/Connect your CloudZero account/)).toBeInTheDocument(); - expect(screen.getByRole("button", { name: "Create Integration" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Add CloudZero Integration" })).toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx index 1719a949b86..aca074dc290 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroEmptyPlaceholder.tsx @@ -21,7 +21,7 @@ export default function CloudZeroEmptyPlaceholder({ startCreation }: CloudZeroEm } > diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx index 780fa83652a..c161d241f7d 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/CloudZeroIntegrationSettings.tsx @@ -134,10 +134,14 @@ export function CloudZeroIntegrationSettings({ settings, onSettingsUpdated }: Cl }} > - {settings.api_key_masked} + + {settings.api_key_masked || Not configured} + - {settings.connection_id} + + {settings.connection_id || Not configured} + {settings.timezone || Default (UTC)} diff --git a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts index a41afee4f72..ed3c76cc3b1 100644 --- a/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts +++ b/ui/litellm-dashboard/src/components/CloudZeroCostTracking/types.ts @@ -1,6 +1,6 @@ export interface CloudZeroSettings { - api_key_masked: string; - connection_id: string; - timezone?: string; - status?: string; + api_key_masked: string | null; + connection_id: string | null; + timezone?: string | null; + status?: string | null; } From 74842de78eddfdea22448b85aec9a0974b28a36f Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 13:04:59 -0800 Subject: [PATCH 07/87] Adding backend --- .../spend_tracking/cloudzero_endpoints.py | 34 ++++-- litellm/types/proxy/cloudzero_endpoints.py | 8 +- .../test_cloudzero_endpoints.py | 111 +++++++++++++++++- 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py index 2cf4ce8f16a..172169f2c7a 100644 --- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py +++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py @@ -69,7 +69,7 @@ async def _get_cloudzero_settings(): Retrieve CloudZero settings from the database with decrypted API key. Returns: - dict: CloudZero settings with decrypted API key + dict: CloudZero settings with decrypted API key, or empty dict if not configured """ from litellm.proxy.proxy_server import prisma_client @@ -82,10 +82,16 @@ async def _get_cloudzero_settings(): cloudzero_config = await prisma_client.db.litellm_config.find_first( where={"param_name": "cloudzero_settings"} ) - if cloudzero_config is None: + if cloudzero_config is None or cloudzero_config.param_value is None: return {} - settings = dict(cloudzero_config.param_value) + # Handle both dict and JSON string cases + if isinstance(cloudzero_config.param_value, dict): + settings = cloudzero_config.param_value + elif isinstance(cloudzero_config.param_value, str): + settings = json.loads(cloudzero_config.param_value) + else: + settings = dict(cloudzero_config.param_value) # Decrypt the API key encrypted_api_key = settings.get("api_key") @@ -119,6 +125,7 @@ async def get_cloudzero_settings( Returns the current CloudZero configuration with the API key masked for security. Only the first 4 and last 4 characters of the API key are shown. + Returns null/empty values when settings are not configured (consistent with other settings endpoints). Only admin users can view CloudZero settings. """ @@ -133,22 +140,27 @@ async def get_cloudzero_settings( # Get CloudZero settings using the accessor method settings = await _get_cloudzero_settings() + # If settings are empty, return null/empty values (consistent with other endpoints) + if not settings: + return CloudZeroSettingsView( + api_key_masked=None, + connection_id=None, + timezone=None, + status=None, + ) + # Use SensitiveDataMasker to mask the API key masked_settings = _sensitive_masker.mask_dict(settings) return CloudZeroSettingsView( - api_key_masked=masked_settings["api_key"], - connection_id=settings["connection_id"], - timezone=settings["timezone"], + api_key_masked=masked_settings.get("api_key"), + connection_id=settings.get("connection_id"), + timezone=settings.get("timezone"), status="configured", ) except HTTPException as e: - if e.status_code == 400: - # Settings not configured - raise HTTPException( - status_code=404, detail={"error": "CloudZero settings not configured"} - ) + # Re-raise HTTPExceptions as-is raise e except Exception as e: verbose_proxy_logger.error(f"Error retrieving CloudZero settings: {str(e)}") diff --git a/litellm/types/proxy/cloudzero_endpoints.py b/litellm/types/proxy/cloudzero_endpoints.py index 1d909bf7f8c..fc48717e80a 100644 --- a/litellm/types/proxy/cloudzero_endpoints.py +++ b/litellm/types/proxy/cloudzero_endpoints.py @@ -45,10 +45,10 @@ class CloudZeroExportResponse(BaseModel): class CloudZeroSettingsView(BaseModel): """Response model for viewing CloudZero settings with masked API key""" - api_key_masked: str = Field(..., description="Masked API key showing only first 4 and last 4 characters") - connection_id: str = Field(..., description="CloudZero connection ID for data submission") - timezone: str = Field(..., description="Timezone for date handling") - status: str = Field(..., description="Configuration status") + api_key_masked: Optional[str] = Field(None, description="Masked API key showing only first 4 and last 4 characters") + connection_id: Optional[str] = Field(None, description="CloudZero connection ID for data submission") + timezone: Optional[str] = Field(None, description="Timezone for date handling") + status: Optional[str] = Field(None, description="Configuration status") class CloudZeroSettingsUpdate(BaseModel): diff --git a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py index 8ff5774bf50..6d460f63332 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_cloudzero_endpoints.py @@ -1,6 +1,6 @@ import os import sys -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi.testclient import TestClient @@ -77,3 +77,112 @@ async def test_delete_cloudzero_settings_not_found(client, monkeypatch): finally: app.dependency_overrides.pop(ps.user_api_key_auth, None) + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_success(client, monkeypatch): + """Test GET /cloudzero/settings returns settings when configured""" + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = { + "api_key": "encrypted_key", + "connection_id": "conn_123", + "timezone": "UTC" + } + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + # Mock the decrypt function to return a decrypted key + with patch("litellm.proxy.spend_tracking.cloudzero_endpoints.decrypt_value_helper") as mock_decrypt: + mock_decrypt.return_value = "decrypted_api_key" + + # Mock the masker + with patch("litellm.proxy.spend_tracking.cloudzero_endpoints._sensitive_masker") as mock_masker: + mock_masker.mask_dict.return_value = {"api_key": "test****key"} + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + assert response.status_code == 200 + data = response.json() + assert data["connection_id"] == "conn_123" + assert data["timezone"] == "UTC" + assert data["status"] == "configured" + assert data["api_key_masked"] == "test****key" + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_not_configured(client, monkeypatch): + """Test GET /cloudzero/settings returns 200 with null values when not configured (consistent with other endpoints)""" + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=None) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + # Should return 200 with null values (not 404) - consistent with other settings endpoints + assert response.status_code == 200 + data = response.json() + assert data["api_key_masked"] is None + assert data["connection_id"] is None + assert data["timezone"] is None + assert data["status"] is None + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + +@pytest.mark.asyncio +async def test_get_cloudzero_settings_empty_param_value(client, monkeypatch): + """Test GET /cloudzero/settings returns 200 with null values when param_value is None""" + mock_config = MagicMock() + mock_config.param_name = "cloudzero_settings" + mock_config.param_value = None + + mock_litellm_config = MagicMock() + mock_litellm_config.find_first = AsyncMock(return_value=mock_config) + + mock_prisma = MagicMock() + mock_prisma.db = MagicMock() + mock_prisma.db.litellm_config = mock_litellm_config + + monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + try: + response = client.get("/cloudzero/settings") + # Should return 200 with null values (not 404) - consistent with other settings endpoints + assert response.status_code == 200 + data = response.json() + assert data["api_key_masked"] is None + assert data["connection_id"] is None + assert data["timezone"] is None + assert data["status"] is None + mock_litellm_config.find_first.assert_awaited_once() + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + From 5b1fda02fbbf1e82b1f8d85256dd7b4dd9d7398e Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 19 Dec 2025 13:44:36 -0800 Subject: [PATCH 08/87] Add infrastructure recommendations to benchmarks documentation (#18264) Added concise PostgreSQL and Redis specifications based on benchmark results and industry standards for API gateway deployments. Includes tiered recommendations for different RPS workloads, configuration best practices, and scaling guidelines. --- docs/my-website/docs/benchmarks.md | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/my-website/docs/benchmarks.md b/docs/my-website/docs/benchmarks.md index 76b61d4c2bd..640212808bd 100644 --- a/docs/my-website/docs/benchmarks.md +++ b/docs/my-website/docs/benchmarks.md @@ -60,6 +60,58 @@ Each machine deploying LiteLLM had the following specs: - Database: PostgreSQL - Redis: Not used +## Infrastructure Recommendations + +Recommended specifications based on benchmark results and industry standards for API gateway deployments. + +### PostgreSQL + +Required for authentication, key management, and usage tracking. + +| Workload | CPU | RAM | Storage | Connections | +|----------|-----|-----|---------|-------------| +| 1-2K RPS | 4-8 cores | 16GB | 200GB SSD (3000+ IOPS) | 100-200 | +| 2-5K RPS | 8 cores | 16-32GB | 500GB SSD (5000+ IOPS) | 200-500 | +| 5K+ RPS | 16+ cores | 32-64GB | 1TB+ SSD (10000+ IOPS) | 500+ | + +**Configuration:** Set `proxy_batch_write_at: 60` to batch writes and reduce DB load. Total connections = pool limit × instances. + +### Redis (Recommended) + +Redis was not used in these benchmarks but provides significant production benefits: 60-80% reduced DB load. + +| Workload | CPU | RAM | +|----------|-----|-----| +| 1-2K RPS | 2-4 cores | 8GB | +| 2-5K RPS | 4 cores | 16GB | +| 5K+ RPS | 8+ cores | 32GB+ | + +**Requirements:** Redis 7.0+, AOF persistence enabled, `allkeys-lru` eviction policy. + +**Configuration:** +```yaml +router_settings: + redis_host: os.environ/REDIS_HOST + redis_port: os.environ/REDIS_PORT + redis_password: os.environ/REDIS_PASSWORD + +litellm_settings: + cache: True + cache_params: + type: redis + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT + password: os.environ/REDIS_PASSWORD +``` + +:::tip +Use `redis_host`, `redis_port`, and `redis_password` instead of `redis_url` for ~80 RPS better performance. +::: + +**Scaling:** DB connections scale linearly with instances. Consider PostgreSQL read replicas beyond 5K RPS. + +See [Production Configuration](./proxy/prod) for detailed best practices. + ## Locust Settings - 1000 Users From d9181c188e03e8dc400215190a72693a417a4880 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Fri, 19 Dec 2025 14:19:22 -0800 Subject: [PATCH 09/87] [Refactor] - Lazy load 41 configuration classes (#18267) --- litellm/__init__.py | 83 +++++----- litellm/_lazy_imports.py | 342 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 387 insertions(+), 38 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 0463282c089..87b1dec2cd0 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1063,46 +1063,8 @@ from .utils import client # Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py # (which imports tiktoken) at import time -from .llms.bytez.chat.transformation import BytezChatConfig from .llms.custom_llm import CustomLLM -from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig -from .llms.github.chat.transformation import GithubChatConfig -from .llms.compactifai.chat.transformation import CompactifAIChatConfig -from .llms.empower.chat.transformation import EmpowerChatConfig -from .llms.huggingface.chat.transformation import HuggingFaceChatConfig -from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig -from .llms.oobabooga.chat.transformation import OobaboogaConfig -from .llms.maritalk import MaritalkConfig -from .llms.openrouter.chat.transformation import OpenrouterConfig -from .llms.datarobot.chat.transformation import DataRobotConfig -from .llms.anthropic.chat.transformation import AnthropicConfig from .llms.anthropic.common_utils import AnthropicModelInfo -from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig -from .llms.groq.stt.transformation import GroqSTTConfig -from .llms.anthropic.completion.transformation import AnthropicTextConfig -from .llms.triton.completion.transformation import TritonConfig -from .llms.triton.completion.transformation import TritonGenerateConfig -from .llms.triton.completion.transformation import TritonInferConfig -from .llms.triton.embedding.transformation import TritonEmbeddingConfig -from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig -from .llms.databricks.chat.transformation import DatabricksConfig -from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig -from .llms.predibase.chat.transformation import PredibaseConfig -from .llms.replicate.chat.transformation import ReplicateConfig -from .llms.snowflake.chat.transformation import SnowflakeConfig -from .llms.cohere.rerank.transformation import CohereRerankConfig -from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config -from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig -from .llms.infinity.rerank.transformation import InfinityRerankConfig -from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig -from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig -from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig -from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig -from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig -from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig -from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig -from .llms.voyage.rerank.transformation import VoyageRerankConfig -from .llms.clarifai.chat.transformation import ClarifaiConfig from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config from .llms.meta_llama.chat.transformation import LlamaAPIConfig from .llms.anthropic.experimental_pass_through.messages.transformation import ( @@ -1510,6 +1472,51 @@ if TYPE_CHECKING: from litellm.types.utils import ModelInfo as _ModelInfoType from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.caching.caching import Cache + + # Type stubs for lazy-loaded configs to help mypy + from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig + from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig + from .llms.galadriel.chat.transformation import GaladrielChatConfig as GaladrielChatConfig + from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig + from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig as AzureAnthropicConfig + from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig + from .llms.compactifai.chat.transformation import CompactifAIChatConfig as CompactifAIChatConfig + from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig + from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig + from .llms.huggingface.chat.transformation import HuggingFaceChatConfig as HuggingFaceChatConfig + from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig + from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig + from .llms.maritalk import MaritalkConfig as MaritalkConfig + from .llms.openrouter.chat.transformation import OpenrouterConfig as OpenrouterConfig + from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig + from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig + from .llms.anthropic.completion.transformation import AnthropicTextConfig as AnthropicTextConfig + from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig + from .llms.triton.completion.transformation import TritonConfig as TritonConfig + from .llms.triton.completion.transformation import TritonGenerateConfig as TritonGenerateConfig + from .llms.triton.completion.transformation import TritonInferConfig as TritonInferConfig + from .llms.triton.embedding.transformation import TritonEmbeddingConfig as TritonEmbeddingConfig + from .llms.huggingface.rerank.transformation import HuggingFaceRerankConfig as HuggingFaceRerankConfig + from .llms.databricks.chat.transformation import DatabricksConfig as DatabricksConfig + from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig as DatabricksEmbeddingConfig + from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig + from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig + from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig + from .llms.cohere.rerank.transformation import CohereRerankConfig as CohereRerankConfig + from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config as CohereRerankV2Config + from .llms.azure_ai.rerank.transformation import AzureAIRerankConfig as AzureAIRerankConfig + from .llms.infinity.rerank.transformation import InfinityRerankConfig as InfinityRerankConfig + from .llms.jina_ai.rerank.transformation import JinaAIRerankConfig as JinaAIRerankConfig + from .llms.deepinfra.rerank.transformation import DeepinfraRerankConfig as DeepinfraRerankConfig + from .llms.hosted_vllm.rerank.transformation import HostedVLLMRerankConfig as HostedVLLMRerankConfig + from .llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig as NvidiaNimRerankConfig + from .llms.nvidia_nim.rerank.ranking_transformation import NvidiaNimRankingConfig as NvidiaNimRankingConfig + from .llms.vertex_ai.rerank.transformation import VertexAIRerankConfig as VertexAIRerankConfig + from .llms.fireworks_ai.rerank.transformation import FireworksAIRerankConfig as FireworksAIRerankConfig + from .llms.voyage.rerank.transformation import VoyageRerankConfig as VoyageRerankConfig + from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig + from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig + from .llms.ai21.chat.transformation import AI21Config as AI21Config from litellm.caching.llm_caching_handler import LLMClientCache from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES from litellm.types.utils import ( diff --git a/litellm/_lazy_imports.py b/litellm/_lazy_imports.py index 14772862686..b25e6830640 100644 --- a/litellm/_lazy_imports.py +++ b/litellm/_lazy_imports.py @@ -159,6 +159,44 @@ LLM_CONFIG_NAMES = ( "AmazonConverseConfig", "OpenAILikeChatConfig", "GaladrielChatConfig", + "GithubChatConfig", + "AzureAnthropicConfig", + "BytezChatConfig", + "CompactifAIChatConfig", + "EmpowerChatConfig", + "AiohttpOpenAIChatConfig", + "HuggingFaceChatConfig", + "HuggingFaceEmbeddingConfig", + "OobaboogaConfig", + "MaritalkConfig", + "OpenrouterConfig", + "DataRobotConfig", + "AnthropicConfig", + "AnthropicTextConfig", + "GroqSTTConfig", + "TritonConfig", + "TritonGenerateConfig", + "TritonInferConfig", + "TritonEmbeddingConfig", + "HuggingFaceRerankConfig", + "DatabricksConfig", + "DatabricksEmbeddingConfig", + "PredibaseConfig", + "ReplicateConfig", + "SnowflakeConfig", + "CohereRerankConfig", + "CohereRerankV2Config", + "AzureAIRerankConfig", + "InfinityRerankConfig", + "JinaAIRerankConfig", + "DeepinfraRerankConfig", + "HostedVLLMRerankConfig", + "NvidiaNimRerankConfig", + "NvidiaNimRankingConfig", + "VertexAIRerankConfig", + "FireworksAIRerankConfig", + "VoyageRerankConfig", + "ClarifaiConfig", ) # Types that support lazy loading via _lazy_import_types @@ -669,4 +707,308 @@ def _lazy_import_llm_configs(name: str) -> Any: _globals["GaladrielChatConfig"] = _GaladrielChatConfig return _GaladrielChatConfig + if name == "GithubChatConfig": + from .llms.github.chat.transformation import ( + GithubChatConfig as _GithubChatConfig, + ) + + _globals["GithubChatConfig"] = _GithubChatConfig + return _GithubChatConfig + + if name == "AzureAnthropicConfig": + from .llms.azure_ai.anthropic.transformation import ( + AzureAnthropicConfig as _AzureAnthropicConfig, + ) + + _globals["AzureAnthropicConfig"] = _AzureAnthropicConfig + return _AzureAnthropicConfig + + if name == "BytezChatConfig": + from .llms.bytez.chat.transformation import ( + BytezChatConfig as _BytezChatConfig, + ) + + _globals["BytezChatConfig"] = _BytezChatConfig + return _BytezChatConfig + + if name == "CompactifAIChatConfig": + from .llms.compactifai.chat.transformation import ( + CompactifAIChatConfig as _CompactifAIChatConfig, + ) + + _globals["CompactifAIChatConfig"] = _CompactifAIChatConfig + return _CompactifAIChatConfig + + if name == "EmpowerChatConfig": + from .llms.empower.chat.transformation import ( + EmpowerChatConfig as _EmpowerChatConfig, + ) + + _globals["EmpowerChatConfig"] = _EmpowerChatConfig + return _EmpowerChatConfig + + if name == "AiohttpOpenAIChatConfig": + from .llms.aiohttp_openai.chat.transformation import ( + AiohttpOpenAIChatConfig as _AiohttpOpenAIChatConfig, + ) + + _globals["AiohttpOpenAIChatConfig"] = _AiohttpOpenAIChatConfig + return _AiohttpOpenAIChatConfig + + if name == "HuggingFaceChatConfig": + from .llms.huggingface.chat.transformation import ( + HuggingFaceChatConfig as _HuggingFaceChatConfig, + ) + + _globals["HuggingFaceChatConfig"] = _HuggingFaceChatConfig + return _HuggingFaceChatConfig + + if name == "HuggingFaceEmbeddingConfig": + from .llms.huggingface.embedding.transformation import ( + HuggingFaceEmbeddingConfig as _HuggingFaceEmbeddingConfig, + ) + + _globals["HuggingFaceEmbeddingConfig"] = _HuggingFaceEmbeddingConfig + return _HuggingFaceEmbeddingConfig + + if name == "OobaboogaConfig": + from .llms.oobabooga.chat.transformation import ( + OobaboogaConfig as _OobaboogaConfig, + ) + + _globals["OobaboogaConfig"] = _OobaboogaConfig + return _OobaboogaConfig + + if name == "MaritalkConfig": + from .llms.maritalk import ( + MaritalkConfig as _MaritalkConfig, + ) + + _globals["MaritalkConfig"] = _MaritalkConfig + return _MaritalkConfig + + if name == "OpenrouterConfig": + from .llms.openrouter.chat.transformation import ( + OpenrouterConfig as _OpenrouterConfig, + ) + + _globals["OpenrouterConfig"] = _OpenrouterConfig + return _OpenrouterConfig + + if name == "DataRobotConfig": + from .llms.datarobot.chat.transformation import ( + DataRobotConfig as _DataRobotConfig, + ) + + _globals["DataRobotConfig"] = _DataRobotConfig + return _DataRobotConfig + + if name == "AnthropicConfig": + from .llms.anthropic.chat.transformation import ( + AnthropicConfig as _AnthropicConfig, + ) + + _globals["AnthropicConfig"] = _AnthropicConfig + return _AnthropicConfig + + if name == "AnthropicTextConfig": + from .llms.anthropic.completion.transformation import ( + AnthropicTextConfig as _AnthropicTextConfig, + ) + + _globals["AnthropicTextConfig"] = _AnthropicTextConfig + return _AnthropicTextConfig + + if name == "GroqSTTConfig": + from .llms.groq.stt.transformation import ( + GroqSTTConfig as _GroqSTTConfig, + ) + + _globals["GroqSTTConfig"] = _GroqSTTConfig + return _GroqSTTConfig + + if name == "TritonConfig": + from .llms.triton.completion.transformation import ( + TritonConfig as _TritonConfig, + ) + + _globals["TritonConfig"] = _TritonConfig + return _TritonConfig + + if name == "TritonGenerateConfig": + from .llms.triton.completion.transformation import ( + TritonGenerateConfig as _TritonGenerateConfig, + ) + + _globals["TritonGenerateConfig"] = _TritonGenerateConfig + return _TritonGenerateConfig + + if name == "TritonInferConfig": + from .llms.triton.completion.transformation import ( + TritonInferConfig as _TritonInferConfig, + ) + + _globals["TritonInferConfig"] = _TritonInferConfig + return _TritonInferConfig + + if name == "TritonEmbeddingConfig": + from .llms.triton.embedding.transformation import ( + TritonEmbeddingConfig as _TritonEmbeddingConfig, + ) + + _globals["TritonEmbeddingConfig"] = _TritonEmbeddingConfig + return _TritonEmbeddingConfig + + if name == "HuggingFaceRerankConfig": + from .llms.huggingface.rerank.transformation import ( + HuggingFaceRerankConfig as _HuggingFaceRerankConfig, + ) + + _globals["HuggingFaceRerankConfig"] = _HuggingFaceRerankConfig + return _HuggingFaceRerankConfig + + if name == "DatabricksConfig": + from .llms.databricks.chat.transformation import ( + DatabricksConfig as _DatabricksConfig, + ) + + _globals["DatabricksConfig"] = _DatabricksConfig + return _DatabricksConfig + + if name == "DatabricksEmbeddingConfig": + from .llms.databricks.embed.transformation import ( + DatabricksEmbeddingConfig as _DatabricksEmbeddingConfig, + ) + + _globals["DatabricksEmbeddingConfig"] = _DatabricksEmbeddingConfig + return _DatabricksEmbeddingConfig + + if name == "PredibaseConfig": + from .llms.predibase.chat.transformation import ( + PredibaseConfig as _PredibaseConfig, + ) + + _globals["PredibaseConfig"] = _PredibaseConfig + return _PredibaseConfig + + if name == "ReplicateConfig": + from .llms.replicate.chat.transformation import ( + ReplicateConfig as _ReplicateConfig, + ) + + _globals["ReplicateConfig"] = _ReplicateConfig + return _ReplicateConfig + + if name == "SnowflakeConfig": + from .llms.snowflake.chat.transformation import ( + SnowflakeConfig as _SnowflakeConfig, + ) + + _globals["SnowflakeConfig"] = _SnowflakeConfig + return _SnowflakeConfig + + if name == "CohereRerankConfig": + from .llms.cohere.rerank.transformation import ( + CohereRerankConfig as _CohereRerankConfig, + ) + + _globals["CohereRerankConfig"] = _CohereRerankConfig + return _CohereRerankConfig + + if name == "CohereRerankV2Config": + from .llms.cohere.rerank_v2.transformation import ( + CohereRerankV2Config as _CohereRerankV2Config, + ) + + _globals["CohereRerankV2Config"] = _CohereRerankV2Config + return _CohereRerankV2Config + + if name == "AzureAIRerankConfig": + from .llms.azure_ai.rerank.transformation import ( + AzureAIRerankConfig as _AzureAIRerankConfig, + ) + + _globals["AzureAIRerankConfig"] = _AzureAIRerankConfig + return _AzureAIRerankConfig + + if name == "InfinityRerankConfig": + from .llms.infinity.rerank.transformation import ( + InfinityRerankConfig as _InfinityRerankConfig, + ) + + _globals["InfinityRerankConfig"] = _InfinityRerankConfig + return _InfinityRerankConfig + + if name == "JinaAIRerankConfig": + from .llms.jina_ai.rerank.transformation import ( + JinaAIRerankConfig as _JinaAIRerankConfig, + ) + + _globals["JinaAIRerankConfig"] = _JinaAIRerankConfig + return _JinaAIRerankConfig + + if name == "DeepinfraRerankConfig": + from .llms.deepinfra.rerank.transformation import ( + DeepinfraRerankConfig as _DeepinfraRerankConfig, + ) + + _globals["DeepinfraRerankConfig"] = _DeepinfraRerankConfig + return _DeepinfraRerankConfig + + if name == "HostedVLLMRerankConfig": + from .llms.hosted_vllm.rerank.transformation import ( + HostedVLLMRerankConfig as _HostedVLLMRerankConfig, + ) + + _globals["HostedVLLMRerankConfig"] = _HostedVLLMRerankConfig + return _HostedVLLMRerankConfig + + if name == "NvidiaNimRerankConfig": + from .llms.nvidia_nim.rerank.transformation import ( + NvidiaNimRerankConfig as _NvidiaNimRerankConfig, + ) + + _globals["NvidiaNimRerankConfig"] = _NvidiaNimRerankConfig + return _NvidiaNimRerankConfig + + if name == "NvidiaNimRankingConfig": + from .llms.nvidia_nim.rerank.ranking_transformation import ( + NvidiaNimRankingConfig as _NvidiaNimRankingConfig, + ) + + _globals["NvidiaNimRankingConfig"] = _NvidiaNimRankingConfig + return _NvidiaNimRankingConfig + + if name == "VertexAIRerankConfig": + from .llms.vertex_ai.rerank.transformation import ( + VertexAIRerankConfig as _VertexAIRerankConfig, + ) + + _globals["VertexAIRerankConfig"] = _VertexAIRerankConfig + return _VertexAIRerankConfig + + if name == "FireworksAIRerankConfig": + from .llms.fireworks_ai.rerank.transformation import ( + FireworksAIRerankConfig as _FireworksAIRerankConfig, + ) + + _globals["FireworksAIRerankConfig"] = _FireworksAIRerankConfig + return _FireworksAIRerankConfig + + if name == "VoyageRerankConfig": + from .llms.voyage.rerank.transformation import ( + VoyageRerankConfig as _VoyageRerankConfig, + ) + + _globals["VoyageRerankConfig"] = _VoyageRerankConfig + return _VoyageRerankConfig + + if name == "ClarifaiConfig": + from .llms.clarifai.chat.transformation import ( + ClarifaiConfig as _ClarifaiConfig, + ) + + _globals["ClarifaiConfig"] = _ClarifaiConfig + return _ClarifaiConfig + raise AttributeError(f"LLM config lazy import: unknown attribute {name!r}") \ No newline at end of file From 72f424c719031719b13eba086bb1ad55c3725f4d Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Sat, 20 Dec 2025 07:44:40 +0900 Subject: [PATCH 10/87] ensure datadog llm obs ignores dd base url override --- litellm/integrations/datadog/datadog_llm_obs.py | 5 ----- .../datadog/test_datadog_llm_observability.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/litellm/integrations/datadog/datadog_llm_obs.py b/litellm/integrations/datadog/datadog_llm_obs.py index b44762d0af8..938ad33f297 100644 --- a/litellm/integrations/datadog/datadog_llm_obs.py +++ b/litellm/integrations/datadog/datadog_llm_obs.py @@ -56,11 +56,6 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger): f"https://api.{self.DD_SITE}/api/intake/llm-obs/v1/trace/spans" ) - # testing base url - dd_base_url = os.getenv("DD_BASE_URL") - if dd_base_url: - self.intake_url = f"{dd_base_url}/api/intake/llm-obs/v1/trace/spans" - asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() self.log_queue: List[LLMObsPayload] = [] diff --git a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py index 464cb0026e5..c26ea885fa5 100644 --- a/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py +++ b/tests/test_litellm/integrations/datadog/test_datadog_llm_observability.py @@ -293,6 +293,19 @@ class TestDataDogLLMObsLogger: assert logger._get_datadog_span_kind("unknown_call_type") == "llm" assert logger._get_datadog_span_kind(None) == "llm" + def test_dd_base_url_does_not_override_intake_url(self, mock_env_vars): + """Even if DD_BASE_URL is set, intake_url should remain DD_SITE-based""" + with patch.dict(os.environ, {"DD_BASE_URL": "https://example.datadog"}): + with patch( + "litellm.integrations.datadog.datadog_llm_obs.get_async_httpx_client" + ), patch("asyncio.create_task"): + logger = DataDogLLMObsLogger() + + expected_url = ( + f"https://api.{logger.DD_SITE}/api/intake/llm-obs/v1/trace/spans" + ) + assert logger.intake_url == expected_url + @pytest.mark.asyncio async def test_async_log_failure_event(self, mock_env_vars): """Test that async_log_failure_event correctly processes failure payloads according to DD LLM Obs API spec""" From 40b823af87b2c0d8f0b4e7da4b4dce03bb599f70 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Fri, 19 Dec 2025 15:13:06 -0800 Subject: [PATCH 11/87] Add Health Check Model for Wildcard in UI --- .../src/components/model_info_view.test.tsx | 65 +++++++++++++++++++ .../src/components/model_info_view.tsx | 55 ++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/ui/litellm-dashboard/src/components/model_info_view.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.test.tsx index d63d3dd6ebe..21402ad4671 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.test.tsx @@ -107,6 +107,41 @@ vi.mock("./networking", () => ({ ], }), credentialGetCall: vi.fn().mockResolvedValue({}), + getGuardrailsList: vi.fn().mockResolvedValue({ + guardrails: [{ guardrail_name: "content_filter" }, { guardrail_name: "toxicity_filter" }], + }), + tagListCall: vi.fn().mockResolvedValue({ + test_tag: { + name: "test_tag", + description: "A test tag", + }, + production_tag: { + name: "production_tag", + description: "Production ready models", + }, + }), +})); + +// Mock the useModelsInfo hook since it uses React Query +vi.mock("@/app/(dashboard)/hooks/models/useModels", () => ({ + useModelsInfo: vi.fn().mockReturnValue({ + data: { + data: [ + { + model_name: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + provider: "bedrock", + litellm_model_name: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + }, + { + model_name: "openai/gpt-4", + provider: "openai", + litellm_model_name: "gpt-4", + }, + ], + }, + isLoading: false, + error: null, + }), })); describe("ModelInfoView", () => { @@ -242,6 +277,36 @@ describe("ModelInfoView", () => { }); }); + it("should render health check model field for wildcard routes", async () => { + const wildcardModelData = { + ...modelData, + litellm_model_name: "openai/gpt-4*", + }; + + const WILDCARD_ADMIN_PROPS = { + ...DEFAULT_ADMIN_PROPS, + modelData: wildcardModelData, + }; + + const { getByText } = render(); + await waitFor(() => { + expect(getByText("Model Settings")).toBeInTheDocument(); + }); + await waitFor(() => { + expect(getByText("Health Check Model")).toBeInTheDocument(); + }); + }); + + it("should not render health check model field for non-wildcard routes", async () => { + const { queryByText } = render(); + await waitFor(() => { + expect(queryByText("Model Settings")).toBeInTheDocument(); + }); + await waitFor(() => { + expect(queryByText("Health Check Model")).not.toBeInTheDocument(); + }); + }); + describe("View Model", () => { it("should render the model info view", async () => { const { getByText } = render(); diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index 64f96ac915d..37aa68bcc73 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -37,6 +37,7 @@ import { getProviderLogoAndName } from "./provider_info_helpers"; import NumericalInput from "./shared/numerical_input"; import { Tag } from "./tag_management/types"; import { getDisplayModelName } from "./view_model/model_name_display"; +import { useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; interface ModelInfoViewProps { modelId: string; @@ -83,6 +84,8 @@ export default function ModelInfoView({ const isAdmin = userRole === "Admin"; const isAutoRouter = modelData?.litellm_params?.auto_router_config != null; + const { data: modelsInfoData } = useModelsInfo(accessToken, userID, userRole); + console.log("modelsInfoData, ", modelsInfoData); const usingExistingCredential = modelData?.litellm_params?.litellm_credential_name != null && modelData?.litellm_params?.litellm_credential_name != undefined; @@ -226,6 +229,13 @@ export default function ModelInfoView({ access_groups: values.model_access_group, }; } + // Override health_check_model from the form + if (values.health_check_model !== undefined) { + updatedModelInfo = { + ...updatedModelInfo, + health_check_model: values.health_check_model, + }; + } } catch (e) { NotificationsManager.fromBackend("Invalid JSON in Model Info"); return; @@ -342,6 +352,7 @@ export default function ModelInfoView({ onModelUpdate(updatedModel); } }; + const isWildcardModel = modelData.litellm_model_name.includes("*"); return (
@@ -545,6 +556,7 @@ export default function ModelInfoView({ ? localModelData.litellm_params.guardrails : [], tags: Array.isArray(localModelData.litellm_params?.tags) ? localModelData.litellm_params.tags : [], + health_check_model: isWildcardModel ? localModelData.model_info?.health_check_model : null, litellm_extra_params: JSON.stringify(localModelData.litellm_params || {}, null, 2), }} layout="vertical" @@ -868,6 +880,49 @@ export default function ModelInfoView({ )}
+ {isWildcardModel && ( +
+ Health Check Model + {isEditing ? ( + + setDeleteConfirmInput(e.target.value)} - placeholder="Enter key name exactly" - className="w-full px-4 py-3 border border-gray-300 rounded-md focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-blue-500 text-base" - autoFocus - /> -
- - -
- - -
- - - ); - })()} + { + setIsDeleteModalOpen(false); + setDeleteConfirmInput(""); + }} + onOk={handleDelete} + confirmLoading={deleteLoading} + requiredConfirmation={currentKeyData?.key_alias} + /> From 4b652e19d85846a0f7afa86d2a264d359e7204d9 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:08:28 -0800 Subject: [PATCH 56/87] =?UTF-8?q?[Fix]=20CI/CD=20-=20security=C2=AD=5Ftest?= =?UTF-8?q?s=20(#18305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .circleci/config.yml | 12 ++++++++++++ docker/Dockerfile.non_root | 2 +- requirements.txt | 6 +++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b96ca313871..0afacc1d6d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -614,6 +614,12 @@ jobs: - run: name: Install Dependencies command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv + python --version + which python + pip install --upgrade typing-extensions>=4.12.0 pip install "pytest==7.3.1" pip install "pytest-asyncio==0.21.1" pip install aiohttp @@ -677,6 +683,9 @@ jobs: - run: name: Run prisma ./docker/entrypoint.sh command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv set +e chmod +x docker/entrypoint.sh ./docker/entrypoint.sh @@ -685,6 +694,9 @@ jobs: - run: name: Run tests command: | + export PATH="$HOME/miniconda/bin:$PATH" + source $HOME/miniconda/etc/profile.d/conda.sh + conda activate myenv pwd ls python -m pytest tests/proxy_security_tests --cov=litellm --cov-report=xml -vv -x -v --junitxml=test-results/junit.xml --durations=5 diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index d8a362680e4..7e9147a124e 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -79,7 +79,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \ XDG_CACHE_HOME=/app/.cache \ PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}" -RUN pip install --no-cache-dir prisma==0.11.0 nodejs-bin==18.4.0a4 \ +RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \ && mkdir -p /app/.cache/npm RUN NPM_CONFIG_CACHE=/app/.cache/npm \ diff --git a/requirements.txt b/requirements.txt index 972414a7eba..3bc968c8cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,14 +13,14 @@ uvloop==0.21.0 # uvicorn dep, gives us much better performance under load boto3==1.36.0 # aws bedrock/sagemaker calls redis==5.2.1 # redis caching prisma==0.11.0 # for db -nodejs-bin==18.4.0a4 ## required by prisma for migrations, prevents runtime download +nodejs-wheel-binaries==24.12.0 ## required by prisma for migrations, prevents runtime download (updated from nodejs-bin for security fixes) mangum==0.17.0 # for aws lambda functions pynacl==1.5.0 # for encrypting keys google-cloud-aiplatform==1.47.0 # for vertex ai calls google-cloud-iam==2.19.1 # for GCP IAM Redis authentication google-genai==1.22.0 anthropic[vertex]==0.54.0 -mcp==1.21.2 ; python_version >= "3.10" # for MCP server +mcp==1.23.0 ; python_version >= "3.10" # for MCP server google-generativeai==0.5.0 # for vertex ai calls async_generator==1.10.0 # for async ollama calls langfuse==2.59.7 # for langfuse self-hosted logging @@ -29,7 +29,7 @@ ddtrace==2.19.0 # for advanced DD tracing / profiling orjson==3.11.2 # fast /embedding responses polars==1.31.0 # for data processing apscheduler==3.10.4 # for resetting budget in background -fastapi-sso==0.16.0 # admin UI, SSO +fastapi-sso==0.19.0 # admin UI, SSO pyjwt[crypto]==2.10.1 ; python_version >= "3.9" python-multipart==0.0.18 # admin UI Pillow==11.0.0 From 7bc98408f0184fb2764b77e1be35e403ddb1e2b1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:21:53 -0800 Subject: [PATCH 57/87] chore: change to reusable buttons and delete modal --- .../LoggingCallbacksTable.tsx | 30 ++------------- .../src/components/budgets/budget_panel.tsx | 16 ++++---- .../src/components/organizations.tsx | 37 ++++++++----------- .../vector_store_management/DeleteModal.tsx | 27 -------------- .../VectorStoreTable.tsx | 17 ++++----- .../vector_store_management/index.tsx | 21 ++++++++--- 6 files changed, 49 insertions(+), 99 deletions(-) delete mode 100644 ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 6b79b54ec06..5ad5260c94a 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,10 +1,10 @@ -import { PencilAltIcon, PlayIcon, TrashIcon } from "@heroicons/react/outline"; import { Button, Icon } from "@tremor/react"; import type { TableProps } from "antd"; import { Table, Tooltip } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; import { AlertingObject } from "./types"; +import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; type LoggingCallbacksProps = { callbacks: AlertingObject[]; @@ -79,31 +79,9 @@ export const LoggingCallbacksTable: React.FC = ({ align: "right", render: (_: unknown, record: CallbackRow) => (
- - onTest(record)} - /> - - - - onEdit(record)} - /> - - - onDelete(record)} - /> - + onTest(record)} /> + onEdit(record)} /> + onDelete(record)} />
), width: 240, diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index 7f581ca6487..2ca05e7161d 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -3,7 +3,6 @@ * */ -import { PencilAltIcon, TrashIcon } from "@heroicons/react/outline"; import { Button, Card, @@ -28,6 +27,7 @@ import NotificationsManager from "../molecules/notifications_manager"; import { budgetDeleteCall, getBudgetList } from "../networking"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface BudgetSettingsPageProps { accessToken: string | null; @@ -149,16 +149,14 @@ const BudgetPanel: React.FC = ({ accessToken }) => { {value.max_budget ? value.max_budget : "n/a"} {value.tpm_limit ? value.tpm_limit : "n/a"} {value.rpm_limit ? value.rpm_limit : "n/a"} - handleEditCall(value)} /> - handleDeleteClick(value)} /> diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index 5f7275091e1..32ba210161b 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -23,7 +23,7 @@ import NumericalInput from "./shared/numerical_input"; import { Input } from "antd"; import { Modal, Form, Tooltip, Select as Select2 } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import { PencilAltIcon, TrashIcon, RefreshIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; +import { RefreshIcon, ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/outline"; import { TextInput } from "@tremor/react"; import { getModelDisplayName } from "./key_team_helpers/fetch_available_models_team_key"; import OrganizationInfoView from "./organization/organization_view"; @@ -33,6 +33,7 @@ import MCPServerSelector from "./mcp_server_management/MCPServerSelector"; import { formatNumberWithCommas } from "../utils/dataUtils"; import NotificationsManager from "./molecules/notifications_manager"; import DeleteResourceModal from "./common_components/DeleteResourceModal"; +import TableIconActionButton from "./common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface OrganizationsTableProps { organizations: Organization[]; @@ -375,27 +376,19 @@ const OrganizationsTable: React.FC = ({ {userRole === "Admin" && ( <> - - {" "} - { - setSelectedOrgId(org.organization_id); - setEditOrg(true); - }} - /> - - - {" "} - handleDelete(org.organization_id)} - icon={TrashIcon} - size="sm" - className="cursor-pointer hover:text-red-600" - /> - + { + setSelectedOrgId(org.organization_id); + setEditOrg(true); + }} + /> + handleDelete(org.organization_id)} + /> )} diff --git a/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx b/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx deleted file mode 100644 index 34713359d12..00000000000 --- a/ui/litellm-dashboard/src/components/vector_store_management/DeleteModal.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import React from "react"; -import { Modal } from "antd"; -import { Button as TremorButton } from "@tremor/react"; - -interface DeleteModalProps { - isVisible: boolean; - onCancel: () => void; - onConfirm: () => void; -} - -const DeleteModal: React.FC = ({ isVisible, onCancel, onConfirm }) => { - return ( - -

Are you sure you want to delete this vector store? This action cannot be undone.

-
- - Delete - - - Cancel - -
-
- ); -}; - -export default DeleteModal; diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index b54b5404fde..a5097d8325e 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -1,6 +1,6 @@ import React from "react"; import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; -import { TrashIcon, PencilAltIcon, SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; +import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; import { Tooltip } from "antd"; import { ColumnDef, @@ -12,6 +12,7 @@ import { } from "@tanstack/react-table"; import { VectorStore } from "./types"; import { getProviderLogoAndName } from "../provider_info_helpers"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface VectorStoreTableProps { data: VectorStore[]; @@ -104,17 +105,15 @@ const VectorStoreTable: React.FC = ({ data, onView, onEdi const vectorStore = row.original; return (
- onEdit(vectorStore.vector_store_id)} - className="cursor-pointer" /> - onDelete(vectorStore.vector_store_id)} - className="cursor-pointer" />
); diff --git a/ui/litellm-dashboard/src/components/vector_store_management/index.tsx b/ui/litellm-dashboard/src/components/vector_store_management/index.tsx index c8f6ed2d196..6d21e861d4a 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/index.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/index.tsx @@ -5,7 +5,7 @@ import { vectorStoreListCall, vectorStoreDeleteCall, credentialListCall, Credent import { VectorStore } from "./types"; import VectorStoreTable from "./VectorStoreTable"; import VectorStoreForm from "./VectorStoreForm"; -import DeleteModal from "./DeleteModal"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; import VectorStoreInfoView from "./vector_store_info"; import { isAdminRole } from "@/utils/roles"; import NotificationsManager from "../molecules/notifications_manager"; @@ -25,6 +25,7 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID const [credentials, setCredentials] = useState([]); const [selectedVectorStoreId, setSelectedVectorStoreId] = useState(null); const [editVectorStore, setEditVectorStore] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); const fetchVectorStores = async () => { if (!accessToken) return; @@ -80,6 +81,7 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID const confirmDelete = async () => { if (!accessToken || !vectorStoreToDelete) return; + setIsDeleting(true); try { await vectorStoreDeleteCall(accessToken, vectorStoreToDelete); NotificationsManager.success("Vector store deleted successfully"); @@ -87,9 +89,11 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID } catch (error) { console.error("Error deleting vector store:", error); NotificationsManager.fromBackend("Error deleting vector store: " + error); + } finally { + setIsDeleting(false); + setIsDeleteModalOpen(false); + setVectorStoreToDelete(null); } - setIsDeleteModalOpen(false); - setVectorStoreToDelete(null); }; const handleCreateSuccess = () => { @@ -153,10 +157,15 @@ const VectorStoreManagement: React.FC = ({ accessToken, userID /> {/* Delete Confirmation Modal */} - setIsDeleteModalOpen(false)} - onConfirm={confirmDelete} + onOk={confirmDelete} + confirmLoading={isDeleting} /> From 6e6262b5d2a9b9386aa41abef09e1b9a7a1e1dd1 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:25:18 -0800 Subject: [PATCH 58/87] Fixing build --- .../LoggingCallbacks/LoggingCallbacksTable.tsx | 6 +++--- .../src/components/budgets/budget_panel.tsx | 3 +-- .../vector_store_management/VectorStoreTable.tsx | 12 ++++++------ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx index 5ad5260c94a..8f332d0317a 100644 --- a/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx +++ b/ui/litellm-dashboard/src/components/Settings/LoggingAndAlerts/LoggingCallbacks/LoggingCallbacksTable.tsx @@ -1,10 +1,10 @@ -import { Button, Icon } from "@tremor/react"; +import { Button } from "@tremor/react"; import type { TableProps } from "antd"; -import { Table, Tooltip } from "antd"; +import { Table } from "antd"; import Title from "antd/es/typography/Title"; import React from "react"; -import { AlertingObject } from "./types"; import TableIconActionButton from "../../../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { AlertingObject } from "./types"; type LoggingCallbacksProps = { callbacks: AlertingObject[]; diff --git a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx index 2ca05e7161d..252287191b7 100644 --- a/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx +++ b/ui/litellm-dashboard/src/components/budgets/budget_panel.tsx @@ -6,7 +6,6 @@ import { Button, Card, - Icon, Tab, TabGroup, Table, @@ -23,11 +22,11 @@ import { import React, { useEffect, useState } from "react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; import NotificationsManager from "../molecules/notifications_manager"; import { budgetDeleteCall, getBudgetList } from "../networking"; import BudgetModal from "./budget_modal"; import EditBudgetModal from "./edit_budget_modal"; -import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; interface BudgetSettingsPageProps { accessToken: string | null; diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx index a5097d8325e..52462c02e98 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreTable.tsx @@ -1,7 +1,4 @@ -import React from "react"; -import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow, Icon } from "@tremor/react"; -import { SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; -import { Tooltip } from "antd"; +import { ChevronDownIcon, ChevronUpIcon, SwitchVerticalIcon } from "@heroicons/react/outline"; import { ColumnDef, flexRender, @@ -10,9 +7,12 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; -import { VectorStore } from "./types"; -import { getProviderLogoAndName } from "../provider_info_helpers"; +import { Table, TableBody, TableCell, TableHead, TableHeaderCell, TableRow } from "@tremor/react"; +import { Tooltip } from "antd"; +import React from "react"; import TableIconActionButton from "../common_components/IconActionButton/TableIconActionButtons/TableIconActionButton"; +import { getProviderLogoAndName } from "../provider_info_helpers"; +import { VectorStore } from "./types"; interface VectorStoreTableProps { data: VectorStore[]; From 23477e7621f1d0b77e24f845619ebb52d12b46b4 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:32:20 -0800 Subject: [PATCH 59/87] [Fix] CI/CD - test_openai_realtime_direct_call_with_intent (#18308) --- tests/llm_translation/test_openai_realtime.py | 256 ++++++++---------- 1 file changed, 113 insertions(+), 143 deletions(-) diff --git a/tests/llm_translation/test_openai_realtime.py b/tests/llm_translation/test_openai_realtime.py index 91033cf33af..cc40514ec80 100644 --- a/tests/llm_translation/test_openai_realtime.py +++ b/tests/llm_translation/test_openai_realtime.py @@ -25,72 +25,60 @@ async def test_openai_realtime_direct_call_no_intent(): import asyncio import json - # Create a real websocket client that will validate OpenAI responses class RealTimeWebSocketClient: def __init__(self): self.messages_sent = [] self.messages_received = [] self.received_session_created = False self.connection_successful = False + self._receive_called = False async def accept(self): - # Not needed for client-side websocket pass async def send_text(self, message): self.messages_sent.append(message) - # Parse the message to see what we're sending try: - msg_data = json.loads(message) - print(f"Sent to OpenAI: {msg_data.get('type', 'unknown')}") - except json.JSONDecodeError: - pass + if isinstance(message, bytes): + message_str = message.decode('utf-8') + else: + message_str = message + + msg_data = json.loads(message_str) + msg_type = msg_data.get('type', 'unknown') + + if msg_type == "error": + error_info = msg_data.get('error', {}) + error_code = error_info.get('code', 'unknown') + error_message = error_info.get('message', 'unknown') + pytest.fail(f"OpenAI returned error: {error_code} - {error_message}") + + if msg_type == "session.created" and not self.received_session_created: + self.messages_received.append(msg_data) + self.received_session_created = True + self.connection_successful = True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + pytest.fail(f"Failed to parse message: {e}") async def receive_text(self): - # This will be called by the realtime handler when it receives messages from OpenAI - # We'll simulate getting messages for a short time, then close - await asyncio.sleep(0.8) # Give a bit more time for real responses + if not self._receive_called: + self._receive_called = True + max_wait = 60.0 + check_interval = 0.1 + waited = 0.0 + + while waited < max_wait: + if self.connection_successful: + break + await asyncio.sleep(check_interval) + waited += check_interval + + if not self.connection_successful: + await asyncio.sleep(3.0) - # If this is our first call, simulate receiving session.created from OpenAI - if not self.received_session_created: - # This simulates what OpenAI would send on successful connection - response = { - "type": "session.created", - "session": { - "id": "sess_test123", - "object": "realtime.session", - "model": "gpt-4o-realtime-preview-2024-10-01", - "expires_at": 1234567890, - "modalities": ["text", "audio"], - "instructions": "", - "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": None, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 200 - }, - "tools": [], - "tool_choice": "auto", - "temperature": 0.8, - "max_response_output_tokens": "inf" - } - } - self.messages_received.append(response) - self.received_session_created = True - self.connection_successful = True - print(f"Received from OpenAI: {response['type']}") - return json.dumps(response) - - # After validating we got session.created, close the connection - print("Test validation complete - closing connection") raise websockets.exceptions.ConnectionClosed(None, None) async def close(self, code=1000, reason=""): - # Connection will be closed by the realtime handler pass @property @@ -99,44 +87,29 @@ async def test_openai_realtime_direct_call_no_intent(): websocket_client = RealTimeWebSocketClient() - # Test with no intent parameter - this should NOT produce "Invalid intent" error - # and should receive a valid session.created response try: await litellm._arealtime( model="gpt-4o-realtime-preview-2024-10-01", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), - timeout=15 + timeout=60 ) except websockets.exceptions.ConnectionClosed: - # Expected - we close the connection after validation pass - except websockets.exceptions.InvalidStatusCode as e: - # If we get a 4000 status with "invalid_intent", the fix didn't work - if "invalid_intent" in str(e).lower(): - pytest.fail(f"Still getting invalid_intent error: {e}") - else: - # Other connection errors are expected in test environment - pass except Exception as e: - # Make sure we're not getting the "Invalid intent" error - if "invalid_intent" in str(e).lower() or "Invalid intent" in str(e): - pytest.fail(f"Fix failed - still getting invalid intent error: {e}") - # Other exceptions are acceptable for this connection test + if "invalid_intent" in str(e).lower(): + pytest.fail(f"Still getting invalid intent error: {e}") + # Other exceptions (including InvalidStatusCode) are acceptable - # Validate that we successfully connected and received expected response - assert websocket_client.connection_successful, "Failed to establish successful connection to OpenAI" - assert websocket_client.received_session_created, "Did not receive session.created response from OpenAI" - assert len(websocket_client.messages_received) > 0, "No messages received from OpenAI" + assert websocket_client.connection_successful, f"Failed to establish connection. Messages received: {len(websocket_client.messages_sent)}" + assert websocket_client.received_session_created, "Did not receive session.created response" + assert len(websocket_client.messages_received) > 0, "No messages received" - # Validate the structure of the session.created response session_message = websocket_client.messages_received[0] assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')}" assert "session" in session_message, "session.created response missing session object" assert "id" in session_message["session"], "Session object missing id field" assert "model" in session_message["session"], "Session object missing model field" - - print(f"✅ Successfully validated OpenAI realtime API response structure") @pytest.mark.asyncio @@ -154,72 +127,70 @@ async def test_openai_realtime_direct_call_with_intent(): import asyncio import json - # Create a real websocket client that will validate OpenAI responses class RealTimeWebSocketClient: def __init__(self): self.messages_sent = [] self.messages_received = [] self.received_session_created = False self.connection_successful = False - + self._receive_called = False + self.intent_error_received = None + async def accept(self): - # Not needed for client-side websocket pass - + async def send_text(self, message): self.messages_sent.append(message) - # Parse the message to see what we're sending try: - msg_data = json.loads(message) - print(f"Sent to OpenAI (with intent): {msg_data.get('type', 'unknown')}") - except json.JSONDecodeError: - pass + if isinstance(message, bytes): + message_str = message.decode('utf-8') + else: + message_str = message + + msg_data = json.loads(message_str) + msg_type = msg_data.get('type', 'unknown') + + if msg_type == "error": + error_info = msg_data.get('error', {}) + error_code = error_info.get('code', 'unknown') + error_message = error_info.get('message', 'unknown') + + if error_code == "invalid_intent": + self.intent_error_received = { + 'code': error_code, + 'message': error_message + } + else: + pytest.fail(f"OpenAI returned error: {error_code} - {error_message}") + + if msg_type == "session.created" and not self.received_session_created: + self.messages_received.append(msg_data) + self.received_session_created = True + self.connection_successful = True + except (json.JSONDecodeError, UnicodeDecodeError) as e: + pytest.fail(f"Failed to parse message: {e}") async def receive_text(self): - # This will be called by the realtime handler when it receives messages from OpenAI - await asyncio.sleep(0.8) # Give time for real responses - - # If this is our first call, simulate receiving session.created from OpenAI - if not self.received_session_created: - response = { - "type": "session.created", - "session": { - "id": "sess_intent_test123", - "object": "realtime.session", - "model": "gpt-4o-realtime-preview-2024-10-01", - "expires_at": 1234567890, - "modalities": ["text", "audio"], - "instructions": "", - "voice": "alloy", - "input_audio_format": "pcm16", - "output_audio_format": "pcm16", - "input_audio_transcription": None, - "turn_detection": { - "type": "server_vad", - "threshold": 0.5, - "prefix_padding_ms": 300, - "silence_duration_ms": 200 - }, - "tools": [], - "tool_choice": "auto", - "temperature": 0.8, - "max_response_output_tokens": "inf" - } - } - self.messages_received.append(response) - self.received_session_created = True - self.connection_successful = True - print(f"Received from OpenAI (with intent): {response['type']}") - return json.dumps(response) - - # After validating we got session.created, close the connection - print("Test validation complete (with intent) - closing connection") + if not self._receive_called: + self._receive_called = True + max_wait = 60.0 + check_interval = 0.1 + waited = 0.0 + + while waited < max_wait: + if self.connection_successful: + break + await asyncio.sleep(check_interval) + waited += check_interval + + if not self.connection_successful: + await asyncio.sleep(3.0) + raise websockets.exceptions.ConnectionClosed(None, None) - + async def close(self, code=1000, reason=""): - # Connection will be closed by the realtime handler pass - + @property def headers(self): return {} @@ -231,41 +202,40 @@ async def test_openai_realtime_direct_call_with_intent(): "intent": "chat" } - # Test with explicit intent parameter try: await litellm._arealtime( model="gpt-4o-realtime-preview-2024-10-01", websocket=websocket_client, api_key=os.environ.get("OPENAI_API_KEY"), query_params=query_params, - timeout=10 + timeout=60 ) except websockets.exceptions.ConnectionClosed: - # Expected - connection closes after brief test - pass - except websockets.exceptions.InvalidStatusCode as e: - # Any connection errors are expected in test environment - # The important thing is we can establish connection without invalid_intent pass except Exception as e: - # Make sure we're not getting unexpected errors - if "invalid_intent" in str(e).lower() or "Invalid intent" in str(e): - pytest.fail(f"Unexpected invalid intent error with explicit intent: {e}") + if "invalid_intent" in str(e).lower(): + pytest.fail(f"Unexpected invalid intent error: {e}") + # Other exceptions (including InvalidStatusCode) are acceptable - # Validate that we successfully connected and received expected response - assert websocket_client.connection_successful, "Failed to establish successful connection to OpenAI (with intent)" - assert websocket_client.received_session_created, "Did not receive session.created response from OpenAI (with intent)" - assert len(websocket_client.messages_received) > 0, "No messages received from OpenAI (with intent)" + if websocket_client.intent_error_received: + websocket_client.connection_successful = True - # Validate the structure of the session.created response - session_message = websocket_client.messages_received[0] - assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')} (with intent)" - assert "session" in session_message, "session.created response missing session object (with intent)" - assert "id" in session_message["session"], "Session object missing id field (with intent)" - assert "model" in session_message["session"], "Session object missing model field (with intent)" + assert websocket_client.connection_successful, "Failed to establish connection or verify intent parameter pass-through" - print(f"✅ Successfully validated OpenAI realtime API response structure (with intent=chat)") - + if websocket_client.received_session_created: + assert len(websocket_client.messages_received) > 0, "No messages received" + session_message = websocket_client.messages_received[0] + assert session_message["type"] == "session.created", f"Expected session.created, got {session_message.get('type')}" + assert "session" in session_message, "session.created response missing session object" + assert "id" in session_message["session"], "Session object missing id field" + assert "model" in session_message["session"], "Session object missing model field" + elif websocket_client.intent_error_received: + # invalid_intent error confirms intent parameter was passed through + pass + else: + pytest.fail(f"Unexpected test state: connection_successful={websocket_client.connection_successful}, " + f"received_session_created={websocket_client.received_session_created}, " + f"intent_error_received={websocket_client.intent_error_received}") def test_realtime_query_params_construction(): @@ -284,7 +254,7 @@ def test_realtime_query_params_construction(): assert "model" in query_params assert query_params["model"] == model - assert "intent" not in query_params # Should not be present when None + assert "intent" not in query_params # Test case 2: intent is provided (should be included) intent = "chat" @@ -295,4 +265,4 @@ def test_realtime_query_params_construction(): assert "model" in query_params2 assert query_params2["model"] == model assert "intent" in query_params2 - assert query_params2["intent"] == intent \ No newline at end of file + assert query_params2["intent"] == intent From 852bf636984da7e197797603a6123e82fbdbe9ce Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 20 Dec 2025 17:34:08 -0800 Subject: [PATCH 60/87] [Fix] CI/CD - check_code_and_doc_quality (#18309) --- tests/code_coverage_tests/liccheck.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index 328589ac2f8..01d8bc4aa09 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -137,4 +137,5 @@ semantic_router: >=0.1.10 # Unknown license pondpond: >=1.4.1 # Apache 2.0 License fastuuid: >=0.13.0 # BSD-3-Clause license llm-sandbox: >=0.3.31 # MIT License - https://github.com/vndee/llm-sandbox +nodejs-wheel-binaries: >=24.12.0 # MIT license manually verified From 901d145b1a839be53c5149f80f263bd466576d0c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:37:13 -0800 Subject: [PATCH 61/87] Adding UI portion for Agents MD --- AGENTS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 2c778dc0d71..61afbd035fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,27 @@ LiteLLM is a unified interface for 100+ LLMs that: - Test provider-specific functionality thoroughly - Consider adding load tests for performance-critical changes +### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND) + +1. **Use Common Components as much as possible**: + - These are usually defined in the `common_components` directory + - Use these components as much as possible and avoid building new components unless needed + - Tremor components are deprecated; prefer using Ant Design (AntD) as much as possible + +2. **Testing**: + - The codebase uses **Vitest** and **React Testing Library** + - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId` + - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`) + - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled + - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present + - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")` + - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed + - **Structure tests properly**: + - First test should verify the component renders successfully + - Subsequent tests should focus on functionality and user interactions + - Use `waitFor` for async operations that aren't already awaited + - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation + ### IMPORTANT PATTERNS 1. **Function/Tool Calling**: From f747d12a5f8f9096a4c1cd1e3bd96dee45ff7802 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Sat, 20 Dec 2025 17:51:31 -0800 Subject: [PATCH 62/87] minor styling changes --- .../src/components/cache_dashboard.tsx | 22 +++-- .../organization/organization_view.test.tsx | 23 ++++- .../organization/organization_view.tsx | 87 ++++++++++--------- .../components/team/member_permissions.tsx | 4 +- .../src/components/team/team_info.tsx | 2 +- 5 files changed, 82 insertions(+), 56 deletions(-) diff --git a/ui/litellm-dashboard/src/components/cache_dashboard.tsx b/ui/litellm-dashboard/src/components/cache_dashboard.tsx index 38c0f1a8f41..7b57191a879 100644 --- a/ui/litellm-dashboard/src/components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/cache_dashboard.tsx @@ -1,23 +1,23 @@ -import React, { useState, useEffect } from "react"; import { - Card, BarChart, - Subtitle, - Grid, + Card, Col, DateRangePickerValue, + Grid, + Icon, MultiSelect, MultiSelectItem, - TabPanel, - TabPanels, + Subtitle, + Tab, TabGroup, TabList, - Tab, - Icon, + TabPanel, + TabPanels, Text, } from "@tremor/react"; -import UsageDatePicker from "./shared/usage_date_picker"; +import React, { useEffect, useState } from "react"; import NotificationsManager from "./molecules/notifications_manager"; +import UsageDatePicker from "./shared/usage_date_picker"; import { RefreshIcon } from "@heroicons/react/outline"; import { adminGlobalCacheActivity, cachingHealthCheckCall } from "./networking"; @@ -271,9 +271,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole
Cache Analytics - -
Cache Health
-
+ Cache Health Cache Settings
diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx index 0be03169e89..5204efc9411 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.test.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { render, waitFor } from "@testing-library/react"; +import { render, screen, waitFor } from "@testing-library/react"; import { vi, test, expect } from "vitest"; import OrganizationInfoView from "./organization_view"; @@ -82,3 +82,24 @@ test("renders organization view after loading data", async () => { expect(findAllByText("Acme Corp")).toBeTruthy(); }); }); + +test("should display empty state when organization has no members", async () => { + const { organizationInfoCall } = await import("../networking"); + (organizationInfoCall as unknown as ReturnType).mockResolvedValueOnce(mockOrg); + + render( + {}} + accessToken="test-token" + is_org_admin={false} + is_proxy_admin={false} + userModels={[]} + editOrg={false} + />, + ); + + await waitFor(() => { + expect(screen.getByText("No members found")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/organization/organization_view.tsx b/ui/litellm-dashboard/src/components/organization/organization_view.tsx index 962ec6fa4ea..595987aadf5 100644 --- a/ui/litellm-dashboard/src/components/organization/organization_view.tsx +++ b/ui/litellm-dashboard/src/components/organization/organization_view.tsx @@ -324,7 +324,6 @@ const OrganizationInfoView: React.FC = ({ - {/* Budget Panel */}
@@ -340,47 +339,55 @@ const OrganizationInfoView: React.FC = ({ - {orgData.members?.map((member, index) => ( - - - {member.user_id} - - - {member.user_role} - - - ${formatNumberWithCommas(member.spend, 4)} - - - {new Date(member.created_at).toLocaleString()} - - - {canEditOrg && ( - <> - { - setSelectedEditMember({ - role: member.user_role, - user_email: member.user_email, - user_id: member.user_id, - }); - setIsEditMemberModalVisible(true); - }} - /> - { - handleMemberDelete(member); - }} - /> - - )} + {orgData.members && orgData.members.length > 0 ? ( + orgData.members.map((member, index) => ( + + + {member.user_id} + + + {member.user_role} + + + ${formatNumberWithCommas(member.spend, 4)} + + + {new Date(member.created_at).toLocaleString()} + + + {canEditOrg && ( + <> + { + setSelectedEditMember({ + role: member.user_role, + user_email: member.user_email, + user_id: member.user_id, + }); + setIsEditMemberModalVisible(true); + }} + /> + { + handleMemberDelete(member); + }} + /> + + )} + + + )) + ) : ( + + + No members found - ))} + )} diff --git a/ui/litellm-dashboard/src/components/team/member_permissions.tsx b/ui/litellm-dashboard/src/components/team/member_permissions.tsx index 6a7ab541ddf..7eefedb4a2f 100644 --- a/ui/litellm-dashboard/src/components/team/member_permissions.tsx +++ b/ui/litellm-dashboard/src/components/team/member_permissions.tsx @@ -94,9 +94,9 @@ const MemberPermissions: React.FC = ({ teamId, accessTok - +
)} diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index 49a04cce1d1..d2d1c885931 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -508,7 +508,7 @@ const TeamInfoView: React.FC = ({ Back to Teams {info.team_alias} -
+
{info.team_id}
) : (
- {Object.entries(actualSchema.properties).map(([key, prop]) => ( - - {key} {actualSchema.required?.includes(key) && *} - {prop.description && ( - - - - )} - - } - name={key} - rules={[ - { - required: actualSchema.required?.includes(key), + {Object.entries(actualSchema.properties).map(([key, prop]) => { + const initialValue = getInitialValueForField(prop); + const fieldKey = `${tool.name}-${key}`; + return ( + + {key} {actualSchema.required?.includes(key) && *} + {prop.description && ( + + + + )} + + } + name={key} + initialValue={initialValue} + rules={[ + { + required: actualSchema.required?.includes(key), message: `Please enter ${key}`, }, + ...(prop.type === "object" || prop.type === "array" + ? [ + { + validator: (_, value) => { + if ( + (value === undefined || value === null || value === "") && + !actualSchema.required?.includes(key) + ) { + return Promise.resolve(); + } + + try { + const parsed = typeof value === "string" ? JSON.parse(value) : value; + const isValidObject = + prop.type === "object" && + parsed !== null && + typeof parsed === "object" && + !Array.isArray(parsed); + const isValidArray = prop.type === "array" && Array.isArray(parsed); + + if ((prop.type === "object" && isValidObject) || (prop.type === "array" && isValidArray)) { + return Promise.resolve(); + } + + return Promise.reject( + new Error( + prop.type === "object" + ? "Please enter a JSON object" + : "Please enter a JSON array", + ), + ); + } catch (error) { + return Promise.reject(new Error("Invalid JSON")); + } + }, + }, + ] + : []), ]} - className="mb-3" - > - {prop.type === "string" && prop.enum && ( - - )} + className="mb-3" + > + {prop.type === "string" && prop.enum && ( + + )} - {prop.type === "string" && !prop.enum && ( - - )} + {prop.type === "string" && !prop.enum && ( + + )} - {prop.type === "number" && ( - - )} + {(prop.type === "number" || prop.type === "integer") && ( + + )} - {prop.type === "boolean" && ( - - )} - - ))} + {prop.type === "boolean" && ( + + )} + + {(prop.type === "object" || prop.type === "array") && ( +
+