From c0cab45350dc2f7ace66fdc375c92cb4e672d65f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:14:42 +0000 Subject: [PATCH 1/9] chore(typing): replace Any kwargs unpacking with validated model parsing Swap `Model(**payload)` for `Model.model_validate(payload)` at the seams where the payload comes back untyped, so basedpyright stops widening every target field to Any. None of the models involved override `__init__`, so validation goes through the same core validator either way. Also route UserRepository through its own typed helpers (find_many, update, find_by_id) instead of the raw Prisma table, drop the redundant `_to_model` override signature, and call generate_key_helper_fn with explicit arguments in the SSO callback rather than splatting an untyped dict. Whole-tree basedpyright: reportAny 21481 -> 20834, reportExplicitAny 7258 -> 7252, with every other rule unchanged or lower. --- basedpyright-code-budget.json | 8 +-- .../proxy/hooks/managed_files.py | 8 +-- .../litellm_core_utils/streaming_handler.py | 4 +- .../llms/azure/responses/transformation.py | 4 +- litellm/llms/custom_httpx/llm_http_handler.py | 10 ++-- .../llms/manus/responses/transformation.py | 4 +- .../hooks/user_management_event_hooks.py | 13 ++--- litellm/proxy/management_endpoints/ui_sso.py | 26 +++++---- .../openai_files_endpoints/common_utils.py | 2 +- litellm/repositories/user_repository.py | 57 ++++++++----------- litellm/router.py | 12 ++-- .../complexity_router/complexity_router.py | 2 +- ruff-strict-budget.json | 4 +- type-discipline-budget.json | 6 +- 14 files changed, 76 insertions(+), 84 deletions(-) diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 65142091712..43df27ea2e2 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -1,6 +1,6 @@ { "reportAny": { - "limit": 31903 + "limit": 31256 }, "reportArgumentType": { "limit": 2645 @@ -24,7 +24,7 @@ "limit": 42 }, "reportExplicitAny": { - "limit": 10214 + "limit": 10208 }, "reportFunctionMemberAccess": { "limit": 11 @@ -33,7 +33,7 @@ "limit": 227 }, "reportIncompatibleMethodOverride": { - "limit": 78 + "limit": 77 }, "reportIncompatibleVariableOverride": { "limit": 12 @@ -99,7 +99,7 @@ "limit": 0 }, "reportUnknownArgumentType": { - "limit": 45366 + "limit": 45357 }, "reportUnknownLambdaType": { "limit": 113 diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 8821736d0ff..d57c1a78f3d 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -215,7 +215,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) if result: - return LiteLLM_ManagedFileTable(**result) + return LiteLLM_ManagedFileTable.model_validate(result) ## CHECK DB db_object = await self.prisma_client.db.litellm_managedfiletable.find_first( @@ -223,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) if db_object: - return LiteLLM_ManagedFileTable(**db_object.model_dump()) + return LiteLLM_ManagedFileTable.model_validate(db_object.model_dump()) return None async def delete_unified_file_id( @@ -349,7 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): if isinstance(batch.file_object, str) else batch.file_object ) - batch_obj = LiteLLMBatch(**batch_data) + batch_obj = LiteLLMBatch.model_validate(batch_data) batch_obj.id = batch.unified_object_id batch_objects.append(batch_obj) @@ -382,7 +382,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "flat_model_file_ids": {"hasSome": model_object_ids}, } ) - return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids] + return [OpenAIFileObject.model_validate(file_object.file_object) for file_object in file_ids] async def check_managed_file_id_access( self, data: Dict, user_api_key_dict: UserAPIKeyAuth diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 60dbf7c644a..0ac22b5bb1e 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -131,8 +131,8 @@ class CustomStreamWrapper: self.sent_last_chunk = False self._stream_created_time: float = time.time() - litellm_params: GenericLiteLLMParams = GenericLiteLLMParams( - **self.logging_obj.model_call_details.get("litellm_params", {}) + litellm_params: GenericLiteLLMParams = GenericLiteLLMParams.model_validate( + dict(**self.logging_obj.model_call_details.get("litellm_params", {})) ) self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False self.sent_first_thinking_block = False diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index d0b0dbb070d..1a860cca5a9 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -66,7 +66,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): ) # Create ResponseReasoningItem object from the item data - reasoning_item = ResponseReasoningItem(**item_data) + reasoning_item = ResponseReasoningItem.model_validate(item_data) # Convert back to dict with exclude_none=True to exclude None fields dict_reasoning_item = reasoning_item.model_dump(exclude_none=True) @@ -346,4 +346,4 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError raise AzureOpenAIError(message=raw_response.text, status_code=raw_response.status_code) - return ResponsesAPIResponse(**raw_response_json) + return ResponsesAPIResponse.model_validate(raw_response_json) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ec1301e5923..d6acbaae434 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6184,10 +6184,12 @@ class BaseLLMHTTPHandler: import websockets from websockets.asyncio.client import ClientConnection - litellm_params = GenericLiteLLMParams( - api_base=api_base, - api_key=api_key, - **kwargs, + litellm_params = GenericLiteLLMParams.model_validate( + { + "api_base": api_base, + "api_key": api_key, + **kwargs, + } ) headers = responses_api_provider_config.validate_environment( headers={}, diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 0db53f90330..e6fbbac0563 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -217,7 +217,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" try: - response = ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") response = ResponsesAPIResponse.model_construct(**raw_response_json) @@ -305,7 +305,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): raw_response_json["id"] = f"unknown-{uuid.uuid4().hex[:8]}" try: - response = ResponsesAPIResponse(**raw_response_json) + response = ResponsesAPIResponse.model_validate(raw_response_json) except Exception: verbose_logger.debug(f"Error constructing ResponsesAPIResponse: {raw_response_json}, using model_construct") response = ResponsesAPIResponse.model_construct(**raw_response_json) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index 6122f0594e8..d8cfae5dab0 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -6,8 +6,6 @@ import asyncio from datetime import datetime, timezone from typing import Optional -from pydantic import BaseModel - import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid @@ -16,7 +14,6 @@ from litellm.proxy._types import ( CommonProxyErrors, LiteLLM_AuditLogs, Litellm_EntityType, - LiteLLM_UserTable, LitellmTableNames, NewUserRequest, NewUserResponse, @@ -58,11 +55,11 @@ class UserManagementEventHooks: try: if prisma_client is None: raise Exception(CommonProxyErrors.db_not_connected_error.value) - user_row: BaseModel = await UserRepository(prisma_client).table.find_first( - where={"user_id": response.user_id} - ) - - user_row_litellm_typed = LiteLLM_UserTable(**user_row.model_dump(exclude_none=True)) + if response.user_id is None: + raise Exception("no user_id returned for the newly created user") + user_row_litellm_typed = await UserRepository(prisma_client).find_by_id(response.user_id) + if user_row_litellm_typed is None: + raise Exception(f"no user row found for user_id={response.user_id}") asyncio.create_task( UserManagementEventHooks.create_internal_user_audit_log( user_id=user_row_litellm_typed.user_id, diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 8682b61f910..110e5883485 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -3137,7 +3137,7 @@ class SSOAuthenticationHandler: _default_team_params = deepcopy(litellm.default_team_params) _new_team_request = team_request.model_dump() _new_team_request.update(_default_team_params) - team_request = NewTeamRequest(**_new_team_request) + team_request = NewTeamRequest.model_validate(_new_team_request) return team_request @staticmethod @@ -3271,14 +3271,6 @@ class SSOAuthenticationHandler: # User might not be already created on first generation of key # But if it is, we want their models preferences - default_ui_key_values: Dict[str, Any] = { - "duration": LITELLM_UI_SESSION_DURATION, - "key_max_budget": litellm.max_ui_session_budget, - "aliases": {}, - "config": {}, - "spend": 0, - "team_id": "litellm-dashboard", - } user_defined_values: Optional[SSOUserDefinedValues] = None if user_custom_sso is not None: @@ -3338,10 +3330,20 @@ class SSOAuthenticationHandler: verbose_proxy_logger.info(f"user_defined_values for creating ui key: {user_defined_values}") - default_ui_key_values.update(user_defined_values) - default_ui_key_values["request_type"] = "key" response = await generate_key_helper_fn( - **default_ui_key_values, # type: ignore + request_type="key", + duration=LITELLM_UI_SESSION_DURATION, + key_max_budget=litellm.max_ui_session_budget, + aliases={}, + config={}, + spend=0, + team_id="litellm-dashboard", + models=user_defined_values["models"], + user_id=user_defined_values["user_id"], + user_email=user_defined_values["user_email"], + user_role=user_defined_values["user_role"], + max_budget=user_defined_values["max_budget"], + budget_duration=user_defined_values["budget_duration"], table_name="key", ) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index b2e36188681..2960b031cd3 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -1035,7 +1035,7 @@ async def get_batch_from_database( if isinstance(db_batch_object.file_object, str) else db_batch_object.file_object ) - response = LiteLLMBatch(**batch_data) + response = LiteLLMBatch.model_validate(batch_data) response.id = batch_id # The stored batch object has the raw provider input_file_id. Resolve to unified ID. diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index f0b5dfd8bc5..e93991463f8 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -3,61 +3,56 @@ User repository for database operations on LiteLLM_UserTable. """ import json -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Mapping, Optional, Type from litellm.models.user import LiteLLM_UserTable -from litellm.repositories.base_repository import BaseRepository +from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict + +_JSON_ENCODED_COLUMNS = frozenset({"metadata", "model_spend", "model_max_budget"}) class UserRepository(BaseRepository[LiteLLM_UserTable]): """Repository for user database operations.""" @property - def table(self) -> Any: + def table(self) -> Any: # any-ok: Prisma table actions are reached through the untyped client wrapper return self.prisma_client.db.litellm_usertable @property def model_class(self) -> Type[LiteLLM_UserTable]: return LiteLLM_UserTable - def _to_model(self, record: Any) -> Optional[LiteLLM_UserTable]: + def _to_model(self, record: Optional[DbRecord]) -> Optional[LiteLLM_UserTable]: """Convert a database record to a User model.""" if record is None: return None - data = record.dict() if hasattr(record, "dict") else dict(record) + return LiteLLM_UserTable.model_validate( + { + column: json.loads(value) if column in _JSON_ENCODED_COLUMNS and isinstance(value, str) else value + for column, value in record_to_dict(record).items() + } + ) - json_fields = ["metadata", "model_spend", "model_max_budget"] - for field in json_fields: - if isinstance(data.get(field), str): - data[field] = json.loads(data[field]) - - return LiteLLM_UserTable(**data) - - async def find_by_id(self, user_id: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]: - return await super().find_by_id(user_id, id_field) + async def find_by_id(self, id_value: str, id_field: str = "user_id") -> Optional[LiteLLM_UserTable]: + return await super().find_by_id(id_value, id_field) async def find_by_email(self, user_email: str) -> Optional[LiteLLM_UserTable]: """Find a user by email.""" - records = await self.table.find_many(where={"user_email": user_email}) - if records: - return self._to_model(records[0]) - return None + records = await self.find_many(where={"user_email": user_email}) + return records[0] if records else None async def find_by_sso_id(self, sso_user_id: str) -> Optional[LiteLLM_UserTable]: """Find a user by SSO ID.""" - record = await self.table.find_unique(where={"sso_user_id": sso_user_id}) - return self._to_model(record) + return await self.find_by_id(sso_user_id, id_field="sso_user_id") async def find_by_organization_id(self, organization_id: str) -> List[LiteLLM_UserTable]: """Find all users in an organization.""" - records = await self.table.find_many(where={"organization_id": organization_id}) - return self._to_model_list(records) + return await self.find_many(where={"organization_id": organization_id}) async def find_by_team_id(self, team_id: str) -> List[LiteLLM_UserTable]: """Find all users in a team.""" - records = await self.table.find_many(where={"teams": {"has": team_id}}) - return self._to_model_list(records) + return await self.find_many(where={"teams": {"has": team_id}}) async def count_billable_users(self) -> int: """Number of users that count toward the license seat limit. @@ -86,7 +81,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): max_budget: Optional[float] = None, user_email: Optional[str] = None, models: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, + metadata: Optional[Mapping[str, object]] = None, max_parallel_requests: Optional[int] = None, tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, @@ -96,7 +91,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): object_permission_id: Optional[str] = None, ) -> LiteLLM_UserTable: """Create a new user.""" - data: Dict[str, Any] = {"user_id": user_id} + data: Dict[str, object] = {"user_id": user_id} if user_alias is not None: data["user_alias"] = user_alias if team_id is not None: @@ -149,7 +144,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): max_budget: Optional[float] = None, user_email: Optional[str] = None, models: Optional[List[str]] = None, - metadata: Optional[Dict[str, Any]] = None, + metadata: Optional[Mapping[str, object]] = None, max_parallel_requests: Optional[int] = None, tpm_limit: Optional[int] = None, rpm_limit: Optional[int] = None, @@ -159,7 +154,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): object_permission_id: Optional[str] = None, ) -> Optional[LiteLLM_UserTable]: """Update a user.""" - data: Dict[str, Any] = {} + data: Dict[str, object] = {} if user_alias is not None: data["user_alias"] = user_alias if team_id is not None: @@ -212,11 +207,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): if not await self.exists(user_id, id_field="user_id"): return None - record = await self.table.update( - where={"user_id": user_id}, - data={"teams": {"push": team_id}}, - ) - return self._to_model(record) + return await self.update(user_id, {"teams": {"push": team_id}}, id_field="user_id") async def remove_from_team(self, user_id: str, team_id: str) -> Optional[LiteLLM_UserTable]: """Remove a user from a team. diff --git a/litellm/router.py b/litellm/router.py index ac00cdbc2b0..399bfcdf3e1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8613,9 +8613,9 @@ class Router: deployment = self.get_deployment(model_id=model_id) if deployment is None or self._is_deployment_blocked(deployment): return None - return CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump( - exclude_none=True - ) + return CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) def get_deployment_by_model_group_name(self, model_group_name: str) -> Optional[Deployment]: """ @@ -8755,9 +8755,9 @@ class Router: return None # Get basic credentials - credentials = CredentialLiteLLMParams(**deployment.litellm_params.model_dump(exclude_none=True)).model_dump( - exclude_none=True - ) + credentials = CredentialLiteLLMParams.model_validate( + deployment.litellm_params.model_dump(exclude_none=True) + ).model_dump(exclude_none=True) # Resolve litellm_credential_name to actual credentials if deployment.litellm_params.litellm_credential_name is not None: diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index b43fe0da4ca..1c6a83c6cd9 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -329,7 +329,7 @@ class ComplexityRouter(CustomLogger): # Parse config - always create a new instance to avoid singleton mutation if complexity_router_config: - self.config = ComplexityRouterConfig(**complexity_router_config) + self.config = ComplexityRouterConfig.model_validate(complexity_router_config) else: self.config = ComplexityRouterConfig() diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index dfdc4efe800..6fb3ed748b6 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -24,7 +24,7 @@ "limit": 130 }, "ANN401": { - "limit": 2010 + "limit": 2009 }, "ASYNC230": { "limit": 14 @@ -324,7 +324,7 @@ "limit": 879 }, "UP006": { - "limit": 12138 + "limit": 12135 }, "UP007": { "limit": 2526 diff --git a/type-discipline-budget.json b/type-discipline-budget.json index c9a1b59cc06..8d5161ec7ea 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -1,9 +1,9 @@ { "LIT001": { - "limit": 23253 + "limit": 23250 }, "LIT002": { - "limit": 27427 + "limit": 27425 }, "LIT003": { "limit": 292 @@ -24,6 +24,6 @@ "limit": 1004 }, "LIT009": { - "limit": 2474 + "limit": 2473 } } From 2258dc08aaa682f9a49484af685a91293ff12012 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:26:04 +0000 Subject: [PATCH 2/9] test(repositories): lock in UserRepository JSON column decoding Covers the columns UserRepository._to_model decodes so a narrower or wider column set fails instead of silently changing what callers read back. --- .../repositories/test_repositories.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py index c923b722991..8253308f393 100644 --- a/tests/test_litellm/repositories/test_repositories.py +++ b/tests/test_litellm/repositories/test_repositories.py @@ -1905,6 +1905,28 @@ class TestUserRepositoryExtended: ) assert updated.user_email == "new@example.com" + @pytest.mark.asyncio + async def test_find_by_id_decodes_only_the_json_encoded_columns(self, repo): + repo._prisma_client.db.litellm_usertable._records["user-json"] = { + "user_id": "user-json", + "user_email": "json@example.com", + "user_role": '{"not": "json"}', + "teams": [], + "models": [], + "metadata": '{"department": "engineering"}', + "model_spend": '{"gpt-4": 10.5}', + "model_max_budget": '{"gpt-4": 100.0}', + } + + user = await repo.find_by_id("user-json") + + assert user is not None + assert user.metadata == {"department": "engineering"} + assert user.model_spend == {"gpt-4": 10.5} + assert user.model_max_budget == {"gpt-4": 100.0} + assert user.user_email == "json@example.com" + assert user.user_role == '{"not": "json"}' + class TestProjectRepositoryExtended: @pytest.fixture From 26ace642be31d25c8826c27947fef76060ece375 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 31 Jul 2026 19:48:30 +0000 Subject: [PATCH 3/9] test(proxy): cover the user-created audit hook's database read-back The hook resolves the newly created user through UserRepository and builds the audit entry from that row. Pin both halves: the entry carries the persisted row's fields rather than the /user/new response, and a user id that resolves to nothing produces no entry at all. --- .../hooks/test_user_management_event_hooks.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py diff --git a/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py b/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py new file mode 100644 index 00000000000..102430ab985 --- /dev/null +++ b/tests/test_litellm/proxy/hooks/test_user_management_event_hooks.py @@ -0,0 +1,99 @@ +import asyncio +import json +import sys +from types import SimpleNamespace +from typing import Any, Dict, List, Optional +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm.proxy._types import NewUserRequest, NewUserResponse, UserAPIKeyAuth +from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks + + +class FakeUserTable: + def __init__(self, rows: List[Dict[str, Any]]): + self._rows = rows + + async def find_unique(self, where: Dict[str, Any]) -> Optional[Dict[str, Any]]: + return next( + (row for row in self._rows if all(row.get(key) == value for key, value in where.items())), + None, + ) + + +class FakePrismaClient: + def __init__(self, rows: List[Dict[str, Any]]): + self.db = SimpleNamespace(litellm_usertable=FakeUserTable(rows)) + + +async def _run_created_hook(prisma_client: FakePrismaClient, audit_log: AsyncMock) -> None: + proxy_server = SimpleNamespace( + prisma_client=prisma_client, + litellm_proxy_admin_name="admin-user", + ) + with ( + patch.dict(sys.modules, {"litellm.proxy.proxy_server": proxy_server}), + patch.object(litellm, "store_audit_logs", True), + patch( + "litellm.proxy.hooks.user_management_event_hooks.create_audit_log_for_update", + audit_log, + ), + patch.object( + UserManagementEventHooks, + "async_send_user_invitation_email", + AsyncMock(), + ), + ): + await UserManagementEventHooks.async_user_created_hook( + data=NewUserRequest(user_email="new@example.com", send_invite_email=False), + response=NewUserResponse( + user_id="user-1", + user_email="new@example.com", + key="sk-test", + ), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-user", api_key="sk-admin"), + ) + await asyncio.sleep(0) + + +@pytest.mark.asyncio +async def test_created_hook_audit_logs_the_row_read_back_from_the_database(): + """The audit entry must describe the persisted user row, not the /user/new response.""" + prisma_client = FakePrismaClient( + [ + { + "user_id": "user-1", + "user_email": "new@example.com", + "user_role": "proxy_admin", + "models": ["gpt-4"], + "teams": ["team-a"], + "metadata": '{"source": "api"}', + } + ] + ) + audit_log = AsyncMock() + + await _run_created_hook(prisma_client, audit_log) + + audit_log.assert_awaited_once() + request_data = audit_log.await_args.kwargs["request_data"] + assert request_data.object_id == "user-1" + assert request_data.action == "created" + + updated_values = json.loads(request_data.updated_values) + assert updated_values["user_role"] == "proxy_admin" + assert updated_values["models"] == ["gpt-4"] + assert updated_values["teams"] == ["team-a"] + assert updated_values["metadata"] == {"source": "api"} + + +@pytest.mark.asyncio +async def test_created_hook_skips_the_audit_log_when_no_user_row_exists(): + """A user id that resolves to nothing must not produce an audit entry.""" + audit_log = AsyncMock() + + await _run_created_hook(FakePrismaClient([]), audit_log) + + audit_log.assert_not_awaited() From 594d0d7a0a5513d31e280811dd2eac9603d2f036 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:03:57 -0700 Subject: [PATCH 4/9] feat(proxy)!: gate all mock testing request params behind a single config flag Handling of the client-supplied mock testing params was split across three places with different behavior for each. Three were dropped from every proxy request, two reached the router untouched, and a request that asked for a synthetic failure came back as an ordinary success with nothing to indicate that no failure had been injected Put all six behind one opt-in, general_settings. dangerously_allow_mock_testing_request_params, and reject rather than drop when it is unset, so a fallback drill cannot report a pass for a test that never ran. The rejection names the params it saw and the config key to set, which is also the answer for anyone following the older docs The flag is config-file only. It is deliberately absent from ConfigGeneralSettings, and that absence is what makes /config/update drop it on parse and /config/field/update reject it; the tests pin both so the field cannot be added back for tidiness without the reason surfacing. Enabling it logs a startup warning naming every param it unlocks BREAKING CHANGE: mock_timeout and mock_testing_rate_limit_error now require general_settings.dangerously_allow_mock_testing_request_params to be set in config.yaml. Previously they were accepted unconditionally --- litellm/proxy/proxy_server.py | 33 ++++ litellm/proxy/route_llm_request.py | 61 ++++++-- tests/test_litellm/proxy/test_proxy_server.py | 80 ++++++++++ .../proxy/test_route_llm_request.py | 148 +++++++++++++++--- 4 files changed, 291 insertions(+), 31 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..06e9cc1fbee 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1041,6 +1041,8 @@ async def proxy_startup_event(app: FastAPI): redis_usage_cache=transaction_buffer_redis_cache, ) + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings=general_settings) + ## SEMANTIC TOOL FILTER ## # Read litellm_settings from config for semantic filter initialization try: @@ -7712,6 +7714,37 @@ class ProxyStartupEvent: proxy_logging_obj.startup_event(llm_router=llm_router, redis_usage_cache=redis_usage_cache) + @staticmethod + def _warn_if_mock_testing_params_enabled(general_settings: dict) -> None: + """Announce, loudly, that any caller may inject synthetic failures.""" + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + if general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is not True: + return + + verbose_proxy_logger.warning( + "\n%s\n" + " DANGEROUS SETTING ENABLED\n" + " general_settings.%s = true\n" + "\n" + " Any caller with a valid key on this proxy can now inject synthetic\n" + " failures and latency into their own requests using these body params:\n" + "%s\n" + "\n" + " A request using them consumes a connection and a concurrency slot\n" + " without reaching a provider, and returns an error the caller chose.\n" + "\n" + " Intended for testing fallback chains. Do not leave enabled.\n" + "%s", + "=" * 72, + MOCK_TESTING_CONFIG_KEY, + "\n".join(f" {name}" for name in GATED_MOCK_PARAM_NAMES), + "=" * 72, + ) + @staticmethod def _validate_redis_transaction_buffer_config( general_settings: dict, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1f5aacc2115..aebdbca86ad 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -8,18 +8,24 @@ import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.router_utils.common_utils import _is_proxy_admin_request -# Router-internal mock_testing_* flag names — kept in sync with -# ``litellm.types.router.MockRouterTestingParams`` by the test -# ``test_mock_testing_kwarg_names_matches_dataclass``. Hardcoding (rather -# than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at +# Client-supplied params that make the router or the call path fabricate a +# failure or a delay instead of calling the provider. The ``mock_testing_*`` +# names are kept in sync with ``litellm.types.router.MockRouterTestingParams`` +# by ``test_gated_mock_params_cover_mock_router_testing_params``. Hardcoding +# (rather than deriving via ``dataclasses.fields(MockRouterTestingParams)`` at # import time) avoids a cyclic import: ``litellm.types.router`` imports # back into proxy modules before this module finishes loading. -_MOCK_TESTING_KWARG_NAMES: tuple = ( +GATED_MOCK_PARAM_NAMES: tuple[str, ...] = ( "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ) +MOCK_TESTING_CONFIG_KEY = "dangerously_allow_mock_testing_request_params" + if TYPE_CHECKING: from litellm.router import Router as _Router @@ -169,6 +175,41 @@ def raise_if_required_body_param_missing(route_type: str, data: Mapping[str, obj ) +class MockTestingParamsDisabledError(HTTPException): + def __init__(self, params: tuple[str, ...]): + super().__init__( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ # mutable-ok: HTTPException.detail has no immutable form; same shape as the sibling errors here + "error": ( + f"Mock testing request params are disabled on this proxy: {', '.join(params)}. " + f"An admin can enable them by setting `general_settings.{MOCK_TESTING_CONFIG_KEY}: true` " + "in config.yaml. This setting cannot be changed from the Admin UI or the API." + ) + }, + ) + + +def raise_if_mock_testing_params_disallowed(data: Mapping[str, object], *, allowed: bool) -> None: + """Reject client-supplied mock testing params unless an admin opted in. + + Rejecting (rather than silently dropping) keeps a request that asked for a + synthetic failure from returning a normal success, which reads as a passing + fallback test that never ran. + """ + if allowed: + return + present = tuple(name for name in GATED_MOCK_PARAM_NAMES if name in data) + if present: + raise MockTestingParamsDisabledError(params=present) + + +def mock_testing_params_allowed() -> bool: + """Read the opt-in from the running proxy's ``general_settings``.""" + import litellm.proxy.proxy_server as proxy_server + + return proxy_server.general_settings.get(MOCK_TESTING_CONFIG_KEY, False) is True + + def get_team_id_from_data(data: dict) -> Optional[str]: """ Get the team id from the data's metadata or litellm_metadata params. @@ -381,12 +422,10 @@ async def route_request( await add_shared_session_to_data(data) - # Strip router-internal mock_testing_* flags. Combined with an - # unauthorized fallback in ``router_settings_override`` they let a - # caller deterministically execute requests against restricted - # models. VERIA-44. - for _key in _MOCK_TESTING_KWARG_NAMES: - data.pop(_key, None) + # Gated here rather than alongside the sibling untrusted-param checks in + # ``litellm_pre_call_utils``: the Responses WebSocket route calls + # ``route_request`` directly and never runs ``add_litellm_data_to_request``. + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..192c674771f 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -10678,3 +10678,83 @@ async def test_async_data_generator_forwards_usage_chunk_without_strip_marker(): assert len(data_frames) == 4 assert any('"usage"' in frame and '"completion_tokens":188' in frame.replace(" ", "") for frame in data_frames) assert frames[-1] == "data: [DONE]\n\n" + + +@pytest.mark.asyncio +async def test_config_field_update_rejects_mock_testing_flag(): + """The mock-testing opt-in is deliberately absent from + ``ConfigGeneralSettings`` so that ``/config/field/update`` refuses it. If + someone later adds the field for tidiness, this test fails and tells them + they have just opened an API write path into a config-file-only setting.""" + from fastapi import HTTPException + + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import update_config_general_settings + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key="sk-test", + ) + + with patch.object(proxy_server_module, "prisma_client", MagicMock()): + with pytest.raises(HTTPException) as exc_info: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name=MOCK_TESTING_CONFIG_KEY, + field_value=True, + config_type="general_settings", + ), + user_api_key_dict=admin, + ) + + assert exc_info.value.status_code == 400 + + +def test_config_update_body_drops_mock_testing_flag(): + """``/config/update`` parses its body as ``ConfigYAML``, whose + ``general_settings`` is a ``ConfigGeneralSettings``. Undeclared keys are + dropped on parse, so the flag never reaches the DB by that route either.""" + from litellm.proxy._types import ConfigYAML + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + parsed = ConfigYAML.model_validate({"general_settings": {MOCK_TESTING_CONFIG_KEY: True}}) + + assert parsed.general_settings is not None + assert MOCK_TESTING_CONFIG_KEY not in parsed.general_settings.model_dump(exclude_none=True) + + +def test_startup_warns_when_mock_testing_params_enabled(caplog): + """Enabling the opt-in must announce itself, naming every param it + unlocks — the config key says ``mock_testing`` but the gate also covers + ``mock_timeout`` and ``mock_delay``, so coverage cannot be inferred from + the name alone.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import ( + GATED_MOCK_PARAM_NAMES, + MOCK_TESTING_CONFIG_KEY, + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled( + general_settings={MOCK_TESTING_CONFIG_KEY: True} + ) + + assert MOCK_TESTING_CONFIG_KEY in caplog.text + for param_name in GATED_MOCK_PARAM_NAMES: + assert param_name in caplog.text + + +def test_startup_is_silent_when_mock_testing_params_disabled(caplog): + """A proxy that never set the opt-in must not emit the warning.""" + import logging + + from litellm.proxy.proxy_server import ProxyStartupEvent + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"): + ProxyStartupEvent._warn_if_mock_testing_params_enabled(general_settings={}) + + assert MOCK_TESTING_CONFIG_KEY not in caplog.text diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 93b3ef1cce8..e74693a4f6c 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -8,6 +8,8 @@ sys.path.insert(0, os.path.abspath("../../..")) # Adds the parent directory to from unittest.mock import MagicMock +from fastapi import HTTPException + from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request @@ -471,38 +473,147 @@ async def test_route_request_with_router_settings_override_preserves_existing(): assert call_kwargs["timeout"] == 30 -def test_mock_testing_kwarg_names_matches_dataclass(): - """``_MOCK_TESTING_KWARG_NAMES`` is hardcoded to avoid a cyclic import - against ``litellm.types.router``. This test guards against drift — - if a new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` - the strip list must be updated to keep covering it.""" +def test_gated_mock_params_cover_mock_router_testing_params(): + """``GATED_MOCK_PARAM_NAMES`` is hardcoded to avoid a cyclic import + against ``litellm.types.router``. This test guards against drift — if a + new ``mock_testing_*`` field is added to ``MockRouterTestingParams`` the + gate must be updated to keep covering it. The gate is a superset: it also + covers params consumed outside that dataclass.""" from dataclasses import fields - from litellm.proxy.route_llm_request import _MOCK_TESTING_KWARG_NAMES + from litellm.proxy.route_llm_request import GATED_MOCK_PARAM_NAMES from litellm.types.router import MockRouterTestingParams - assert set(_MOCK_TESTING_KWARG_NAMES) == {f.name for f in fields(MockRouterTestingParams)} + assert {f.name for f in fields(MockRouterTestingParams)} <= set(GATED_MOCK_PARAM_NAMES) + assert {"mock_testing_rate_limit_error", "mock_timeout", "mock_delay"} <= set(GATED_MOCK_PARAM_NAMES) -@pytest.mark.asyncio @pytest.mark.parametrize( - "mock_flag", + "mock_param", [ "mock_testing_fallbacks", "mock_testing_context_fallbacks", "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", ], ) -async def test_route_request_strips_mock_testing_flags(mock_flag): - """VERIA-44: router-internal testing flags must not survive a - user-supplied request body. Without this strip, an attacker can - combine ``mock_testing_fallbacks=true`` with an unauthorized fallback - in ``router_settings_override`` to deterministically execute requests - against restricted models.""" +def test_mock_params_rejected_when_not_allowed(mock_param): + """Every gated param must be rejected by name when the proxy has not + opted in, and the error must point the caller at the config key.""" + from litellm.proxy.route_llm_request import ( + MOCK_TESTING_CONFIG_KEY, + raise_if_mock_testing_params_disallowed, + ) + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + assert exc_info.value.status_code == 400 + error_message = exc_info.value.detail["error"] + assert mock_param in error_message + assert MOCK_TESTING_CONFIG_KEY in error_message + + +@pytest.mark.parametrize( + "mock_param", + [ + "mock_testing_fallbacks", + "mock_testing_context_fallbacks", + "mock_testing_content_policy_fallbacks", + "mock_testing_rate_limit_error", + "mock_timeout", + "mock_delay", + ], +) +def test_mock_params_pass_through_when_allowed(mock_param): + """With the opt-in set, gated params must survive untouched — a gate that + rejects correctly but strips anyway would leave the feature unusable.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = {"model": "gpt-3.5-turbo", mock_param: True} + + raise_if_mock_testing_params_disallowed(data, allowed=True) + + assert data[mock_param] is True + + +def test_mock_param_gate_reports_every_param_present(): + """A request carrying several gated params must name all of them, so a + caller fixing one is not surprised by the next.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + + data = { + "model": "gpt-3.5-turbo", + "mock_testing_fallbacks": True, + "mock_delay": 30, + } + + with pytest.raises(HTTPException) as exc_info: + raise_if_mock_testing_params_disallowed(data, allowed=False) + + error_message = exc_info.value.detail["error"] + assert "mock_testing_fallbacks" in error_message + assert "mock_delay" in error_message + + +def test_ordinary_request_is_not_rejected_by_the_mock_param_gate(): + """The gate must not fire on a request that carries no gated param.""" + from litellm.proxy.route_llm_request import raise_if_mock_testing_params_disallowed + data = { "model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}], - mock_flag: True, + "mock_response": "hi", + } + + raise_if_mock_testing_params_disallowed(data, allowed=False) + + +@pytest.mark.asyncio +async def test_route_request_rejects_mock_params_by_default(monkeypatch): + """End-to-end through ``route_request``: with no opt-in configured the + request is rejected before it ever reaches the router.""" + import litellm.proxy.proxy_server as proxy_server + + monkeypatch.setattr(proxy_server, "general_settings", {}, raising=False) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, + } + llm_router = MagicMock() + + with pytest.raises(HTTPException) as exc_info: + await route_request(data, llm_router, None, "acompletion") + + assert exc_info.value.status_code == 400 + llm_router.acompletion.assert_not_called() + + +@pytest.mark.asyncio +async def test_route_request_forwards_mock_params_when_opted_in(monkeypatch): + """End-to-end through ``route_request``: with the opt-in set the param + reaches the router, which is what makes a fallback drill possible.""" + import litellm.proxy.proxy_server as proxy_server + + from litellm.proxy.route_llm_request import MOCK_TESTING_CONFIG_KEY + + monkeypatch.setattr( + proxy_server, + "general_settings", + {MOCK_TESTING_CONFIG_KEY: True}, + raising=False, + ) + + data = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "mock_testing_fallbacks": True, } llm_router = MagicMock() llm_router.acompletion.return_value = "ok" @@ -510,10 +621,7 @@ async def test_route_request_strips_mock_testing_flags(mock_flag): await route_request(data, llm_router, None, "acompletion") call_kwargs = llm_router.acompletion.call_args[1] - assert mock_flag not in call_kwargs - # The flag is also gone from the original data dict so any subsequent - # processing (e.g. logging) doesn't see it either. - assert mock_flag not in data + assert call_kwargs["mock_testing_fallbacks"] is True @pytest.mark.parametrize("route_type", ["agenerate_content", "agenerate_content_stream"]) From cee78d61eadadfb2b0a3c25b723db357d1e40687 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:38:58 +0000 Subject: [PATCH 5/9] chore(typing): re-ratchet lint budgets after merging staging --- type-discipline-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index b79aae63b0b..05499e83c42 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -3,7 +3,7 @@ "limit": 23250 }, "LIT002": { - "limit": 27280 + "limit": 27277 }, "LIT003": { "limit": 292 From 764b2337701f972ecafd172c9d0705ed976189b6 Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Fri, 31 Jul 2026 17:52:01 -0700 Subject: [PATCH 6/9] refactor(proxy): drop an inaccurate comment on the mock testing gate The comment said the Responses WebSocket route never runs add_litellm_data_to_request. It does, via common_processing_pre_call_logic, so the note recorded a request-flow constraint that does not hold The gate stays in route_request, which is the dispatch chokepoint and where the previous handling lived --- litellm/proxy/route_llm_request.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index aebdbca86ad..12e9857219e 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -422,9 +422,6 @@ async def route_request( await add_shared_session_to_data(data) - # Gated here rather than alongside the sibling untrusted-param checks in - # ``litellm_pre_call_utils``: the Responses WebSocket route calls - # ``route_request`` directly and never runs ``add_litellm_data_to_request``. raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) data.pop("enable_tag_filtering", None) From b4ff05be8edb669c0cb033776c74cfc749e9407c Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Fri, 31 Jul 2026 18:10:48 -0700 Subject: [PATCH 7/9] fix(proxy): stop model writes 500ing on another pod's delete (#35400) * fix(proxy): stop model writes 500ing on another pod's delete A model write judges the reload it triggers by diffing this pod's router before and after, and reports anything that stopped serving as damage. On a pod that has not yet polled a delete another pod made, the snapshot still lists that model; the reload then evicts it because the db no longer has it, and the guard reads its own correct reconcile as degradation. The row is written and served, but the caller gets a 500. Since propagation between pods is a 30s db poll, any delete followed by a create inside that window can land on a pod that has not caught up, so a delete-then-create pair returns 500 whenever the two requests hit different pods. _delete_deployment already computes exactly the set that settles it: the ids the db and config still want. Thread it up through _update_llm_router, add_deployment and clear_cache to the verdict, and intersect the drop set with it so an id the db no longer has stops counting as collateral. Where no reconcile ran the set is None and every drop is still reported, so a genuinely broken reload is caught as before. _delete_deployment now returns that set instead of a delete count; the count had no callers in the proxy, and the tests asserting it already assert the eviction calls. * test(proxy): fold reload-verdict test commentary into docstrings and assertions Greptile flagged the inline comments against the repo's no-new-comments rule. The case-by-case context moves into the test docstring, and the two return-contract assertions carry their reasoning as failure messages instead. * test: fix clear_cache mock return type in model block/unblock tests --- .../model_management_endpoints.py | 48 ++++++++++--- litellm/proxy/proxy_server.py | 39 +++++++---- .../test_model_management_endpoints.py | 70 ++++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 6 +- tests/test_litellm/proxy/test_proxy_server.py | 19 +++-- .../test_update_llm_router_resilience.py | 15 ++-- .../test_litellm/test_model_block_unblock.py | 2 +- 7 files changed, 158 insertions(+), 41 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index b6422d7f5ae..41904e3883b 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -324,7 +324,7 @@ async def patch_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( @@ -344,6 +344,7 @@ async def patch_model( before=live_before_reload, written_models=[(model_id, getattr(updated_model, "model_info", None))], action="update", + still_desired=still_desired_ids, ) return updated_model @@ -429,7 +430,7 @@ async def _set_model_blocked_status( ) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() asyncio.create_task( create_object_audit_log( @@ -450,6 +451,7 @@ async def _set_model_blocked_status( before=live_before_reload, written_models=[(data.model_id, getattr(updated_model, "model_info", None))], action=action, + still_desired=still_desired_ids, ) return updated_model @@ -1355,6 +1357,7 @@ async def add_new_model( """ live_before_reload = live_model_ids_snapshot() + still_desired_ids: frozenset[str] | None = None try: _original_litellm_model_name = model_params.model_name if model_params.model_info.team_id is None: @@ -1369,7 +1372,9 @@ async def add_new_model( user_api_key_dict=user_api_key_dict, prisma_client=prisma_client, ) - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) # don't let failed slack alert block the /model/new response _alerting = general_settings.get("alerting", []) or [] if "slack" in _alerting: @@ -1414,6 +1419,7 @@ async def add_new_model( before=live_before_reload, written_models=[(model_response.model_id, getattr(model_response, "model_info", None))], action="create", + still_desired=still_desired_ids, ) return model_response @@ -1542,7 +1548,7 @@ async def update_model( # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates) live_before_reload = live_model_ids_snapshot() - await clear_cache() + still_desired_ids = await clear_cache() ## CREATE AUDIT LOG ## asyncio.create_task( create_object_audit_log( @@ -1569,6 +1575,7 @@ async def update_model( before=live_before_reload, written_models=[(_model_id, getattr(model_response, "model_info", None))], action="update", + still_desired=still_desired_ids, ) return model_response @@ -1814,6 +1821,7 @@ def reload_serving_verdict( before: frozenset[str], written_models: Sequence[tuple[str, object]], written_must_serve: bool, + still_desired: frozenset[str] | None = None, ) -> tuple[tuple[str, ...], tuple[str, ...]]: """Judge a write-triggered reload by diffing the router's serving state instead of trusting any layer of the reload stack to report its own failure. @@ -1828,10 +1836,14 @@ def reload_serving_verdict( this write and blaming it would block unrelated metadata fixes - not written but live before and gone now: collateral degradation of this pod caused by the reload this request triggered (a wholesale re-add failure, or a - newly introduced conflict), always reported + newly introduced conflict), reported only when the db still wants that id + + ``still_desired`` is the db + config id set the reload just reconciled against. An + id absent from it was deleted on purpose, most often by another pod this one had not + yet polled, so the reload dropping it is the reconcile working rather than damage. + Without it (no reconcile ran) every drop is reported, which is the safe direction. Returns (written ids violating their obligation, collateral ids no longer served). - Best effort under concurrent admin writes: the snapshot spans only this request. """ now = live_model_ids_snapshot() written_ids = frozenset(model_id for model_id, _ in written_models) @@ -1843,7 +1855,8 @@ def reload_serving_verdict( ) else: missing = tuple(model_id for model_id, _ in written_models if model_id in before and model_id not in now) - collateral = tuple(sorted(before - now - written_ids)) + dropped = before - now - written_ids + collateral = tuple(sorted(dropped if still_desired is None else dropped & still_desired)) return (missing, collateral) @@ -1851,12 +1864,18 @@ def raise_if_reload_degraded_serving( before: frozenset[str], written_models: Sequence[tuple[str, object]], action: str, + still_desired: frozenset[str] | None = None, ) -> None: """The caller-visible error this pod's model-write endpoints owe their caller when the model they wrote is not being served after the reload they triggered. The DB write is durable either way and every other pod reloads on its own interval; this speaks only for the handling pod.""" - missing, collateral = reload_serving_verdict(before=before, written_models=written_models, written_must_serve=True) + missing, collateral = reload_serving_verdict( + before=before, + written_models=written_models, + written_must_serve=True, + still_desired=still_desired, + ) if not missing and not collateral: return missing_clause = ( @@ -1882,9 +1901,12 @@ def raise_if_reload_degraded_serving( ) -async def clear_cache(): +async def clear_cache() -> frozenset[str] | None: """ Clear router caches and reload models. + + Returns the db + config id set the reload reconciled against, or None when no + reload ran, so callers can pass it to raise_if_reload_degraded_serving. """ from litellm.proxy.proxy_server import ( llm_router, @@ -1896,7 +1918,7 @@ async def clear_cache(): if llm_router is None or prisma_client is None: verbose_proxy_logger.debug("llm_router or prisma_client is None, skipping cache clear") - return + return None try: # Only clear DB models, preserve config models @@ -1943,10 +1965,14 @@ async def clear_cache(): llm_router.quality_routers.pop(model_name, None) # Reload only DB models - await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await proxy_config.add_deployment( + prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj + ) verbose_proxy_logger.debug( f"Cleared {len(db_model_ids)} DB models, preserved {len(config_models)} config models" ) + return still_desired_ids except Exception as e: verbose_proxy_logger.exception(f"Failed to clear cache and reload models. Due to error - {str(e)}") + return None diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a60ea2da019..15d977b75f2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5300,7 +5300,7 @@ class ProxyConfig: _model_info = RouterModelInfo(id=model.model_id, db_model=db_model) return _model_info - async def _delete_deployment(self, db_models: list) -> int: + async def _delete_deployment(self, db_models: list) -> frozenset[str] | None: """ (Helper function of add deployment) -> combined to reduce prisma db calls @@ -5309,14 +5309,16 @@ class ProxyConfig: - Remove any that are missing Return: - - int - returns number of deleted deployments + - frozenset[str] - the ids the db + config say should be served after this + reconcile, so a caller can tell an id this evicted on purpose from one that + went missing. None when no reconcile ran and that set is therefore unknown. """ global user_config_file_path, llm_router combined_id_list = [] ## BASE CASES ## if llm_router is None: - return 0 + return None # NOTE: db_models may be legitimately empty when all DB models have been deleted. # Do NOT short-circuit on len(db_models) == 0 — we must still evict any # DB-sourced deployments that are no longer in the DB. The caller @@ -5337,7 +5339,7 @@ class ProxyConfig: "Skipping deployment cleanup to avoid removing valid models.", str(e), ) - return 0 + return None model_list = config.get("model_list", None) if model_list: for model in model_list: @@ -5361,13 +5363,10 @@ class ProxyConfig: router_model_ids = llm_router.get_model_ids() # Check for model IDs in llm_router not present in combined_id_list and delete them - deleted_deployments = 0 for model_id in router_model_ids: if model_id not in combined_id_list: - is_deleted = llm_router.delete_deployment(id=model_id) - if is_deleted is not None: - deleted_deployments += 1 - return deleted_deployments + llm_router.delete_deployment(id=model_id) + return frozenset(combined_id_list) def _resolve_db_litellm_param(self, key: str, value: object) -> object: if not isinstance(value, str): @@ -5452,9 +5451,11 @@ class ProxyConfig: self, new_models: Optional[Json], proxy_logging_obj: ProxyLogging, - ): + ) -> frozenset[str] | None: global llm_router, llm_model_list, master_key, general_settings + still_desired_ids: frozenset[str] | None = None + # Load config separately so a timeout here doesn't block model loading config_data: dict = {} search_tools = None @@ -5501,7 +5502,7 @@ class ProxyConfig: if search_tools is not None and llm_router is not None: llm_router.search_tools = search_tools ## DELETE MODEL LOGIC - await self._delete_deployment(db_models=models_list) + still_desired_ids = await self._delete_deployment(db_models=models_list) ## ADD MODEL LOGIC self._add_deployment(db_models=models_list) @@ -5527,6 +5528,8 @@ class ProxyConfig: proxy_logging_obj=proxy_logging_obj, ) + return still_desired_ids + def _add_callback_from_db_to_in_memory_litellm_callbacks( self, callback: str, @@ -6159,14 +6162,20 @@ class ProxyConfig: self, prisma_client: PrismaClient, proxy_logging_obj: ProxyLogging, - ): + ) -> frozenset[str] | None: """ - Check db for new models - Check if model id's in router already - If not, add to router + + Returns the ids the db + config say should be served after the reconcile, or + None when no reconcile ran. Callers that judge their own reload need it to tell + a deliberate eviction from a deployment that went missing. """ global llm_router, llm_model_list, master_key, general_settings + still_desired_ids: frozenset[str] | None = None + try: # warm the config cache so the per-param reads below all hit await prefetch_config_params( @@ -6186,7 +6195,9 @@ class ProxyConfig: new_models = await self._get_models_from_db(prisma_client=prisma_client) # update llm router - await self._update_llm_router(new_models=new_models, proxy_logging_obj=proxy_logging_obj) + still_desired_ids = await self._update_llm_router( + new_models=new_models, proxy_logging_obj=proxy_logging_obj + ) db_general_settings = await get_config_param(prisma_client, "general_settings") @@ -6204,6 +6215,8 @@ class ProxyConfig: "litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {}".format(str(e)) ) + return still_desired_ids + async def _init_non_llm_objects_in_db(self, prisma_client: PrismaClient): """ Use this to read non-llm objects from the db and initialize them diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 1e8add52f74..8dbf38555b3 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -839,7 +839,7 @@ class TestUpdateModel: ), patch( "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", - new=AsyncMock(return_value=True), + new=AsyncMock(return_value=None), ) as mock_clear_cache, ): await update_model( @@ -3223,7 +3223,7 @@ class TestPatchModelBlockedAuthGate: ), patch( "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache", - new=AsyncMock(return_value=True), + new=AsyncMock(return_value=None), ), ): result = await patch_model( @@ -3314,6 +3314,72 @@ class TestWriteSurfacesReloadDrop: before=frozenset({"m-live", "m-collateral"}), written_models=[("m-live", None)], action="update" ) + def test_a_model_the_db_no_longer_has_is_not_collateral(self, monkeypatch): + """Another pod deleting a model is not this pod's reload breaking. + + A pod that has not yet polled the delete still lists the id when the write + snapshots `before`; the reload it triggers then evicts the id because the db no + longer has it. That eviction is the reconcile working, so it must not fail the + write. `still_desired` is the db + config set the reload reconciled against, so + an id missing from it drops out of the collateral diff. + + The cases below, in order: an id the db no longer wants is not collateral and the + write succeeds; an id the db still wants that stopped serving is real degradation + and still raises, so a genuinely broken reload is caught; and with no reconcile at + all the desired set is unknown, so every drop is reported. + """ + import litellm + from litellm.proxy._types import ProxyException + from litellm.proxy.management_endpoints.model_management_endpoints import ( + raise_if_reload_degraded_serving, + reload_serving_verdict, + ) + + live_router = litellm.Router( + model_list=[ + { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "m-live", "db_model": True}, + } + ] + ) + monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", live_router) + + _, collateral = reload_serving_verdict( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + written_must_serve=True, + still_desired=frozenset({"m-live"}), + ) + assert collateral == () + + assert ( + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + action="create", + still_desired=frozenset({"m-live"}), + ) + is None + ) + + with pytest.raises(ProxyException, match="m-should-be-serving"): + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-should-be-serving"}), + written_models=[("m-live", None)], + action="create", + still_desired=frozenset({"m-live", "m-should-be-serving"}), + ) + + with pytest.raises(ProxyException, match="m-deleted-elsewhere"): + raise_if_reload_degraded_serving( + before=frozenset({"m-live", "m-deleted-elsewhere"}), + written_models=[("m-live", None)], + action="create", + still_desired=None, + ) + class TestModelInfoAsMapping: """The model_info column reaches consumers as a dict or as its JSON string; this is diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 8d1d8185e4d..99287b92b8f 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -1404,12 +1404,12 @@ def test_ProxyConfig_get_model_info_with_id_missing_model_id_raises(monkeypatch) @pytest.mark.asyncio -async def test_ProxyConfig__delete_deployment_empty_returns_zero(monkeypatch): +async def test_ProxyConfig__delete_deployment_no_router_returns_none(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None) pc = ProxyConfig() result = await pc._delete_deployment(db_models=[]) - snapshot = {"deleted": result, "router_was": "none", "empty_db_models": True} - assert snapshot == {"deleted": 0, "router_was": "none", "empty_db_models": True} + snapshot = {"still_desired": result, "router_was": "none", "empty_db_models": True} + assert snapshot == {"still_desired": None, "router_was": "none", "empty_db_models": True} @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b9a33bd2cef..1ecf7e8b83a 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -2459,13 +2459,11 @@ async def test_delete_deployment_type_mismatch(): patch("litellm.proxy.proxy_server.user_config_file_path", "test_config.yaml"), ): # Call the function under test - deleted_count = await pc._delete_deployment(db_models=[]) + still_desired = await pc._delete_deployment(db_models=[]) # The two SHA-hash models have no corresponding entry in combined_id_list # and must be evicted. - assert ( - deleted_count == 2 - ), f"Expected 2 deletions (SHA-hash models), got {deleted_count}" + assert len(deleted_ids) == 2, f"Expected 2 deletions (SHA-hash models), got {deleted_ids}" assert ( "a96e12e76b36a57cfae57a41288eb41567629cac89b4828c6f7074afc3534695" in deleted_ids @@ -2485,6 +2483,12 @@ async def test_delete_deployment_type_mismatch(): "12345679" not in deleted_ids ), f"Model 12345679 should NOT be deleted. Deleted IDs: {deleted_ids}" + assert still_desired is not None + assert {"12345678", "12345679"} <= still_desired, ( + "the int-keyed config models must come back as strings in the desired set, so a " + f"caller judging its own reload reads them as wanted rather than evicted; got {still_desired}" + ) + @pytest.mark.asyncio async def test_get_config_from_file(tmp_path, monkeypatch): @@ -9119,10 +9123,13 @@ class TestDeleteDeploymentSync: with patch.object( proxy_config, "get_config", AsyncMock(return_value={"model_list": []}) ): - count = await proxy_config._delete_deployment(db_models=[]) + still_desired = await proxy_config._delete_deployment(db_models=[]) mock_router.delete_deployment.assert_called_once_with(id="model-id-to-evict") - assert count == 1 + assert still_desired == frozenset(), ( + "an empty db and an empty config want nothing, which must stay distinct from " + f"the None returned when no reconcile ran at all; got {still_desired}" + ) @pytest.mark.asyncio async def test_update_llm_router_skips_update_on_db_fetch_failure(self): diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index fd0df4805e6..a7dc9c1783e 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -115,8 +115,10 @@ class TestDeleteDeploymentResilience: """Test _delete_deployment handles get_config failures gracefully.""" @pytest.mark.asyncio - async def test_returns_zero_when_get_config_times_out(self): - """Should return 0 (no deletions) when get_config fails, not raise.""" + async def test_returns_none_when_get_config_times_out(self): + """Should return None (no reconcile ran, desired set unknown) when get_config + fails, not raise. A caller judging its own reload must not read that as "the db + wants nothing" and blame the reload for every model it serves.""" proxy_config = ProxyConfig() db_models = [_make_db_model("gpt-5.1", "db-id-1")] @@ -136,8 +138,8 @@ class TestDeleteDeploymentResilience: ): result = await proxy_config._delete_deployment(db_models=db_models) - # Should safely return 0 instead of raising - assert result == 0 + # Should safely return None instead of raising + assert result is None # Should NOT have deleted any deployments mock_router.delete_deployment.assert_not_called() @@ -175,5 +177,8 @@ class TestDeleteDeploymentResilience: result = await proxy_config._delete_deployment(db_models=db_models) # "stale-id" should have been deleted (not in db_models or config) - assert result == 1 mock_router.delete_deployment.assert_called_once_with(id="stale-id") + assert result == frozenset({"db-id-1", "config-id-1"}), ( + "the returned set must be what the db + config still want, so a caller can " + f"tell that eviction apart from a deployment that went missing; got {result}" + ) diff --git a/tests/test_litellm/test_model_block_unblock.py b/tests/test_litellm/test_model_block_unblock.py index dc0098e405e..ff66bedf0dc 100644 --- a/tests/test_litellm/test_model_block_unblock.py +++ b/tests/test_litellm/test_model_block_unblock.py @@ -36,7 +36,7 @@ def _setup_model_block_mocks(monkeypatch, *, updated_blocked: bool): mock_router = MagicMock() mock_router.get_model_ids.return_value = [model_id] - mock_clear_cache = AsyncMock(return_value=True) + mock_clear_cache = AsyncMock(return_value=None) mock_audit_log = AsyncMock(return_value=None) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) From e38df02d859457e0d1003587e2d2e711c21cca24 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:18:48 -0700 Subject: [PATCH 8/9] fix(ui): show pass through route selections and match team id substrings in team search (#35319) * fix(ui): show pass through route selections in team/key forms and match team id substrings in team search Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * perf(teams): keep team id search index-friendly with a prefix match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(teams): keep /v2/team/list search id matching exact by default and add an opt-in prefix mode Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: milan Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../management_endpoints/team_endpoints.py | 13 ++++- .../management_endpoints/team_endpoints.py | 4 +- .../test_team_endpoints.py | 57 +++++++++++++++++-- .../app/(dashboard)/hooks/teams/useTeams.ts | 3 + ui/litellm-dashboard/src/components/Teams.tsx | 36 ++++++------ .../components/TeamsPage/TeamsTable.test.tsx | 13 +++++ .../src/components/TeamsPage/TeamsTable.tsx | 1 + .../PassThroughRoutesSelector.tsx | 2 +- .../organisms/create_key_button.tsx | 2 - .../src/components/team/TeamInfo.test.tsx | 51 +++++++++++++++++ .../src/components/team/TeamInfo.tsx | 35 ++++++------ .../components/templates/key_edit_view.tsx | 37 ++++++------ ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 +- 13 files changed, 189 insertions(+), 69 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index c35c17aa359..fa01f43d049 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -131,6 +131,7 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkUpdateTeamMemberPermissionsRequest, BulkUpdateTeamMemberPermissionsResponse, GetTeamMemberPermissionsResponse, + TeamIdSearchMatch, TeamListItem, TeamListResponse, TeamMemberAddResult, @@ -3973,6 +3974,7 @@ async def _build_team_list_where_conditions( user_id: Optional[str], use_deleted_table: bool, search: Optional[str] = None, + search_team_id_match: TeamIdSearchMatch = "exact", org_admin_org_ids: Optional[List[str]] = None, user_api_key_cache: Optional[Any] = None, proxy_logging_obj: Optional[Any] = None, @@ -3996,7 +3998,7 @@ async def _build_team_list_where_conditions( if search: where_conditions["OR"] = [ - {"team_id": search}, + ({"team_id": {"startsWith": search}} if search_team_id_match == "prefix" else {"team_id": search}), {"team_alias": {"contains": search, "mode": "insensitive"}}, ] @@ -4230,8 +4232,14 @@ async def list_team_v2( ), search: Optional[str] = fastapi.Query( default=None, - description="Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive).", + description="Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive).", ), + search_team_id_match: Annotated[ + TeamIdSearchMatch, + fastapi.Query( + description="How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match." + ), + ] = "exact", page: int = fastapi.Query(default=1, description="Page number for pagination", ge=1), page_size: int = fastapi.Query(default=10, description="Number of teams per page", ge=1, le=100), sort_by: Optional[str] = fastapi.Query( @@ -4308,6 +4316,7 @@ async def list_team_v2( user_id=user_id, use_deleted_table=use_deleted_table, search=search, + search_team_id_match=search_team_id_match, org_admin_org_ids=org_admin_org_ids, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 0e555535874..2d9387da956 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel @@ -11,6 +11,8 @@ from litellm.proxy._types import ( Member, ) +TeamIdSearchMatch = Literal["exact", "prefix"] + class GetTeamMemberPermissionsRequest(BaseModel): """Request to get the team member permissions for a team""" diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a658a4b7353..25a0ff644f8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -3627,9 +3627,9 @@ async def test_list_team_v2_with_invalid_status(): @pytest.mark.asyncio async def test_list_team_v2_search_builds_or_clause(): """ - `search` should be passed as a Prisma OR across team_id (exact) and - team_alias (case-insensitive contains), so the UI can hit a single - backend filter with either a UUID or a name fragment. + `search` should be passed as a Prisma OR across an exact team_id match and a + case-insensitive team_alias contains, so the UI needs one backend filter. + Exact id matching is the documented default and must not change. """ from unittest.mock import AsyncMock, Mock, patch @@ -3671,6 +3671,54 @@ async def test_list_team_v2_search_builds_or_clause(): } +@pytest.mark.asyncio +async def test_list_team_v2_search_team_id_match_prefix(): + """ + Opting into `search_team_id_match="prefix"` should widen the team_id side of + the search OR to an index-friendly prefix match, so the first characters of a + team id quoted in a proxy error find the team. + """ + from unittest.mock import AsyncMock, Mock, patch + + from fastapi import Request + + from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.management_endpoints.team_endpoints import list_team_v2 + + mock_request = Mock(spec=Request) + mock_admin = UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin_user" + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client: + mock_db = Mock() + mock_prisma_client.db = mock_db + mock_db.litellm_teamtable.find_many = AsyncMock(return_value=[]) + mock_db.litellm_teamtable.count = AsyncMock(return_value=0) + + await list_team_v2( + http_request=mock_request, + user_id=None, + organization_id=None, + team_id=None, + team_alias=None, + search="66c432fa", + search_team_id_match="prefix", + user_api_key_dict=mock_admin, + page=1, + page_size=10, + status=None, + ) + + find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs + assert find_many_kwargs["where"] == { + "OR": [ + {"team_id": {"startsWith": "66c432fa"}}, + {"team_alias": {"contains": "66c432fa", "mode": "insensitive"}}, + ] + } + + @pytest.mark.asyncio async def test_list_team_v2_search_composes_with_user_id_filter(): """ @@ -3724,6 +3772,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): team_id=None, team_alias=None, search="team_a", + search_team_id_match="prefix", user_api_key_dict=mock_user_api_key_dict, page=1, page_size=10, @@ -3733,7 +3782,7 @@ async def test_list_team_v2_search_composes_with_user_id_filter(): find_many_kwargs = mock_db.litellm_teamtable.find_many.call_args.kwargs where = find_many_kwargs["where"] assert where["OR"] == [ - {"team_id": "team_a"}, + {"team_id": {"startsWith": "team_a"}}, {"team_alias": {"contains": "team_a", "mode": "insensitive"}}, ] assert where["team_id"] == {"in": ["team_a", "team_b"]} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts index f532c44ffd7..4061026b94d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/teams/useTeams.ts @@ -24,6 +24,7 @@ export interface TeamListCallOptions { teamID?: string | null; team_alias?: string | null; search?: string | null; + searchTeamIdMatch?: "exact" | "prefix" | null; userID?: string | null; sortBy?: string | null; sortOrder?: string | null; @@ -48,6 +49,7 @@ export const teamListCall = async ( organization_id: options.organizationID, team_alias: options.team_alias, search: options.search, + search_team_id_match: options.searchTeamIdMatch, user_id: options.userID, page, page_size: pageSize, @@ -213,6 +215,7 @@ const deletedTeamListCall = async ( organization_id: options.organizationID, team_alias: options.team_alias, search: options.search, + search_team_id_match: options.searchTeamIdMatch, user_id: options.userID, page, page_size: pageSize, diff --git a/ui/litellm-dashboard/src/components/Teams.tsx b/ui/litellm-dashboard/src/components/Teams.tsx index 0a3c7fc736d..582f75578ed 100644 --- a/ui/litellm-dashboard/src/components/Teams.tsx +++ b/ui/litellm-dashboard/src/components/Teams.tsx @@ -957,25 +957,23 @@ const Teams: React.FC = ({ accessToken, userID, userRole, premiumUser placeholder="Select vector stores (optional)" /> - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes (optional)" - disabled={!premiumUser || !isProxyAdminRole(userRole || "")} - /> - + + diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx index 9469be12128..09bd3e9f245 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.test.tsx @@ -196,6 +196,19 @@ describe("server-side filtering maps controls to the right query params", () => expect(mockUseTeamsTable).toHaveBeenLastCalledWith(1, 50, expect.objectContaining({ search: "platform" })); }); }); + + it("opts into team id prefix matching so a partial id from a proxy error finds the team", async () => { + renderTable(); + fireEvent.change(screen.getByTestId("datatable-search"), { target: { value: "66c432fa" } }); + + await waitFor(() => { + expect(mockUseTeamsTable).toHaveBeenLastCalledWith( + 1, + 50, + expect.objectContaining({ search: "66c432fa", searchTeamIdMatch: "prefix" }), + ); + }); + }); }); describe("non-admin scoping", () => { diff --git a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx index 3b75db52b16..5f5a6f26c0a 100644 --- a/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx +++ b/ui/litellm-dashboard/src/components/TeamsPage/TeamsTable.tsx @@ -66,6 +66,7 @@ export function TeamsTable({ userRole, userID, onSelectTeam, onEditTeam, onDelet team_alias: getFilterValue("alias"), teamID: getFilterValue("team_id"), search: searchQuery.trim() || undefined, + searchTeamIdMatch: "prefix" as const, userID: isAdminView ? undefined : userID ?? undefined, sortBy: sorting[0]?.id, sortOrder: toSortOrder(sorting), diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx index 5cea88b2af2..e02125dea56 100644 --- a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx @@ -3,7 +3,7 @@ import { Select } from "antd"; import { getPassThroughEndpointsCall } from "../networking"; interface PassThroughRoutesSelectorProps { - onChange: (selectedRoutes: string[]) => void; + onChange?: (selectedRoutes: string[]) => void; value?: string[]; className?: string; accessToken: string; diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 711652eb783..00f36f016b7 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -1385,8 +1385,6 @@ const CreateKey: React.FC = ({ team, teams, data, addKey, autoOp } > form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} accessToken={accessToken} placeholder={ !premiumUser diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx index 25365d26a12..712cff80649 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx @@ -18,6 +18,7 @@ vi.mock("@/components/networking", () => ({ getTeamPermissionsCall: vi.fn(), organizationInfoCall: vi.fn(), getRouterSettingsCall: vi.fn().mockResolvedValue({ fields: [] }), + getPassThroughEndpointsCall: vi.fn(), })); vi.mock("@/components/utils/dataUtils", () => ({ @@ -1142,4 +1143,54 @@ describe("TeamInfoView", () => { expect(within(dropdown).getByTitle("opt-in")).toBeInTheDocument(); }); }); + + describe("allowed pass through routes", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["gpt-4"] })); + vi.mocked(networking.teamUpdateCall).mockResolvedValue({ data: {}, team_id: "123" } as any); + vi.mocked(networking.getPassThroughEndpointsCall).mockResolvedValue({ + endpoints: [{ path: "/bedrock-passthrough", methods: ["POST"] }], + }); + }); + + it("should show a route picked from the dropdown in the field and save it", async () => { + const user = userEvent.setup({ delay: null }); + + renderWithProviders(); + + await waitFor(() => { + expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0); + }); + + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + + const routesLabel = await screen.findByText("Allowed Pass Through Routes"); + const routesFormItem = routesLabel.closest(".ant-form-item") as HTMLElement; + + await user.click(within(routesFormItem).getByRole("combobox")); + + const option = await screen.findByTitle("POST /bedrock-passthrough"); + await user.click(option); + + await waitFor(() => { + expect(within(routesFormItem).getByText(/\/bedrock-passthrough/)).toBeInTheDocument(); + }); + + await user.click(screen.getByRole("button", { name: /save changes/i })); + + await waitFor(() => { + expect(networking.teamUpdateCall).toHaveBeenCalledWith( + "test-token", + expect.objectContaining({ + team_id: "123", + metadata: expect.objectContaining({ + allowed_passthrough_routes: ["/bedrock-passthrough"], + }), + }), + ); + }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index acd5a8966a7..34570043f52 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -1361,25 +1361,22 @@ const TeamInfoView: React.FC = ({ /> - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder="Select pass through routes" - disabled={!premiumUser || !is_proxy_admin} - /> - + + diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 962c6bc3568..4b6c266eb8f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -689,26 +689,23 @@ export function KeyEditView({ - - - form.setFieldValue("allowed_passthrough_routes", values)} - value={form.getFieldValue("allowed_passthrough_routes")} - accessToken={accessToken || ""} - placeholder={ - !premiumUser - ? "Premium feature - Upgrade to set allowed pass through routes by key" - : Array.isArray(keyData.metadata?.allowed_passthrough_routes) && - keyData.metadata.allowed_passthrough_routes.length > 0 - ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` - : "Select or enter allowed pass through routes" - } - disabled={!premiumUser} - /> - + + 0 + ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` + : "Select or enter allowed pass through routes" + } + disabled={!premiumUser} + /> diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 9a301f1c474..109d638fb9c 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -58372,8 +58372,10 @@ export interface operations { team_id?: string | null; /** @description Only return teams which this 'team_alias' belongs to. Supports partial matching. */ team_alias?: string | null; - /** @description Combined search: matches teams whose 'team_id' equals the value OR whose 'team_alias' contains it (case-insensitive). */ + /** @description Combined search: matches teams whose 'team_id' matches the value OR whose 'team_alias' contains it (case-insensitive). */ search?: string | null; + /** @description How 'search' matches 'team_id': 'exact' (default) or 'prefix' for a case-sensitive prefix match. */ + search_team_id_match?: "exact" | "prefix"; /** @description Page number for pagination */ page?: number; /** @description Number of teams per page */ From 7c388b1fd97a88d11e7afacc8a4b6b056ae8ffeb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:39:35 +0000 Subject: [PATCH 9/9] refactor(proxy): rename the audit hook's user row variable The typed-vs-raw distinction the _litellm_typed suffix marked is gone now that the repository returns LiteLLM_UserTable directly. --- litellm/proxy/hooks/user_management_event_hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/hooks/user_management_event_hooks.py b/litellm/proxy/hooks/user_management_event_hooks.py index d8cfae5dab0..e40db7f3f1f 100644 --- a/litellm/proxy/hooks/user_management_event_hooks.py +++ b/litellm/proxy/hooks/user_management_event_hooks.py @@ -57,18 +57,18 @@ class UserManagementEventHooks: raise Exception(CommonProxyErrors.db_not_connected_error.value) if response.user_id is None: raise Exception("no user_id returned for the newly created user") - user_row_litellm_typed = await UserRepository(prisma_client).find_by_id(response.user_id) - if user_row_litellm_typed is None: + user_row = await UserRepository(prisma_client).find_by_id(response.user_id) + if user_row is None: raise Exception(f"no user row found for user_id={response.user_id}") asyncio.create_task( UserManagementEventHooks.create_internal_user_audit_log( - user_id=user_row_litellm_typed.user_id, + user_id=user_row.user_id, action="created", litellm_changed_by=user_api_key_dict.user_id, user_api_key_dict=user_api_key_dict, litellm_proxy_admin_name=litellm_proxy_admin_name, before_value=None, - after_value=user_row_litellm_typed.model_dump_json(exclude_none=True), + after_value=user_row.model_dump_json(exclude_none=True), ) ) except Exception as e: