From 60289aa73e1edf0dfce73b902bc3848f85422c02 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 7 Oct 2025 07:12:45 -0700 Subject: [PATCH 01/39] feat(scim_v2.py): if group.id doesn't exist, use external id --- .../management_endpoints/scim/scim_v2.py | 281 ++++++++++-------- 1 file changed, 152 insertions(+), 129 deletions(-) diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index fd7d150a7f4..b8f6b4a4460 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -4,7 +4,6 @@ This is an enterprise feature and requires a premium license. """ -from litellm._uuid import uuid from typing import Any, Dict, List, Optional, Set, Tuple from fastapi import ( @@ -17,11 +16,12 @@ from fastapi import ( Request, Response, ) -from typing_extensions import TypedDict from pydantic import BaseModel +from typing_extensions import TypedDict import litellm from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import ( LiteLLM_TeamTable, @@ -51,32 +51,31 @@ from litellm.types.proxy.management_endpoints.scim_v2 import * class UserProvisionerHelpers: """Helper methods for user provisioning operations.""" - + @staticmethod async def handle_existing_user_by_email( - prisma_client, - new_user_request: NewUserRequest + prisma_client, new_user_request: NewUserRequest ) -> Optional[SCIMUser]: """ Check if a user with the given email already exists and update them if found. - + Args: prisma_client: Database client new_user_request: New user request data - + Returns: SCIMUser if user was updated, None if no existing user found """ if not new_user_request.user_email: return None - + existing_user = await prisma_client.db.litellm_usertable.find_first( where={"user_email": new_user_request.user_email} ) - + if not existing_user: return None - + # Update the user updated_user = await prisma_client.db.litellm_usertable.update( where={"user_id": existing_user.user_id}, @@ -88,12 +87,15 @@ class UserProvisionerHelpers: "metadata": safe_dumps(new_user_request.metadata), }, ) - - return await ScimTransformations.transform_litellm_user_to_scim_user(updated_user) + + return await ScimTransformations.transform_litellm_user_to_scim_user( + updated_user + ) class ScimUserData(TypedDict): """Typed structure for extracted SCIM user data.""" + user_email: Optional[str] user_alias: Optional[str] sso_user_id: Optional[str] @@ -105,6 +107,7 @@ class ScimUserData(TypedDict): class GroupMemberExtractionResult(BaseModel): """Result of extracting and processing group members.""" + existing_member_ids: List[str] created_users: List[NewUserResponse] all_member_ids: List[str] # existing + newly created @@ -121,7 +124,7 @@ scim_router = APIRouter( async def _get_prisma_client_or_raise_exception(): """Check if database is connected and raise HTTPException if not.""" from litellm.proxy.proxy_server import prisma_client - + if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No database connected"}) return prisma_client @@ -130,32 +133,32 @@ async def _get_prisma_client_or_raise_exception(): async def _check_user_exists(user_id: str): """Check if user exists and return user, raise 404 if not found.""" prisma_client = await _get_prisma_client_or_raise_exception() - + user = await prisma_client.db.litellm_usertable.find_unique( where={"user_id": user_id} ) - + if not user: raise HTTPException( status_code=404, detail={"error": f"User not found with ID: {user_id}"} ) - + return user async def _check_team_exists(team_id: str): """Check if team exists and return team, raise 404 if not found.""" prisma_client = await _get_prisma_client_or_raise_exception() - + team = await prisma_client.db.litellm_teamtable.find_unique( where={"team_id": team_id} ) - + if not team: raise HTTPException( status_code=404, detail={"error": f"Group not found with ID: {team_id}"} ) - + return team @@ -184,7 +187,9 @@ def _extract_scim_user_data(user: SCIMUser) -> ScimUserData: } -def _build_scim_metadata(given_name: Optional[str], family_name: Optional[str], active: Optional[bool] = None) -> Dict[str, Any]: +def _build_scim_metadata( + given_name: Optional[str], family_name: Optional[str], active: Optional[bool] = None +) -> Dict[str, Any]: """Build metadata dictionary with SCIM data.""" metadata: Dict[str, Any] = { "scim_metadata": LiteLLM_UserScimMetadata( @@ -192,17 +197,17 @@ def _build_scim_metadata(given_name: Optional[str], family_name: Optional[str], familyName=family_name, ).model_dump() } - + if active is not None: metadata["scim_active"] = active - + return metadata async def _extract_group_member_ids(group: SCIMGroup) -> GroupMemberExtractionResult: """ Extract member IDs from SCIMGroup, creating users that don't exist. - + Returns: GroupMemberExtractionResult with existing members, created users, and all member IDs """ @@ -210,35 +215,34 @@ async def _extract_group_member_ids(group: SCIMGroup) -> GroupMemberExtractionRe existing_member_ids = [] created_users = [] all_member_ids = [] - + if group.members: for member in group.members: user_id = member.value - + # Check if user exists user = await prisma_client.db.litellm_usertable.find_unique( where={"user_id": user_id} ) - + if user: existing_member_ids.append(user_id) all_member_ids.append(user_id) else: # Create the user if they don't exist using our helper created_user = await _create_user_if_not_exists( - user_id=user_id, - created_via="scim_group_membership" + user_id=user_id, created_via="scim_group_membership" ) - + if created_user: created_users.append(created_user) all_member_ids.append(user_id) # If creation failed, user is skipped (logged in helper) - + return GroupMemberExtractionResult( existing_member_ids=existing_member_ids, created_users=created_users, - all_member_ids=all_member_ids + all_member_ids=all_member_ids, ) @@ -246,7 +250,7 @@ async def _get_team_members_display(member_ids: List[str]) -> List[SCIMMember]: """Get SCIMMember objects with display names for a list of member IDs.""" prisma_client = await _get_prisma_client_or_raise_exception() members: List[SCIMMember] = [] - + for member_id in member_ids: user = await prisma_client.db.litellm_usertable.find_unique( where={"user_id": member_id} @@ -254,18 +258,20 @@ async def _get_team_members_display(member_ids: List[str]) -> List[SCIMMember]: if user: display_name = user.user_email or user.user_id members.append(SCIMMember(value=user.user_id, display=display_name)) - + return members -async def _handle_team_membership_changes(user_id: str, existing_teams: List[str], new_teams: List[str]) -> None: +async def _handle_team_membership_changes( + user_id: str, existing_teams: List[str], new_teams: List[str] +) -> None: """Handle adding/removing user from teams based on changes.""" existing_teams_set = set(existing_teams) new_teams_set = set(new_teams) - + teams_to_add = new_teams_set - existing_teams_set teams_to_remove = existing_teams_set - new_teams_set - + if teams_to_add or teams_to_remove: await patch_team_membership( user_id=user_id, @@ -274,19 +280,21 @@ async def _handle_team_membership_changes(user_id: str, existing_teams: List[str ) -async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_group") -> Optional[NewUserResponse]: +async def _create_user_if_not_exists( + user_id: str, created_via: str = "scim_group" +) -> Optional[NewUserResponse]: """ Helper function to create a user if they don't exist. - + Args: user_id: The user ID to create created_via: Context for where the user was created from - + Returns: LiteLLM_UserTable if user was created, None if creation failed """ from litellm.proxy.management_endpoints.internal_user_endpoints import new_user - + try: # Get default role for new internal users default_role: Optional[ @@ -313,7 +321,7 @@ async def _create_user_if_not_exists(user_id: str, created_via: str = "scim_grou created_user = await new_user(data=new_user_request) verbose_proxy_logger.info(f"Created user {user_id} via {created_via}") return created_user - + except Exception as e: verbose_proxy_logger.exception(f"Failed to create user {user_id}: {e}") return None @@ -324,7 +332,7 @@ async def _get_team_member_user_ids_from_team(team: LiteLLM_TeamTable) -> List[s Get the IDs of the members from a team. Use one source of truth for the member IDs: team.members_with_roles - + """ member_user_ids: List[str] = [] for member in team.members_with_roles or []: @@ -337,7 +345,6 @@ async def _get_team_member_user_ids_from_team(team: LiteLLM_TeamTable) -> List[s return member_user_ids - # Dependency to set the correct SCIM Content-Type async def set_scim_content_type(response: Response): """Sets the Content-Type header to application/scim+json""" @@ -450,7 +457,7 @@ async def get_user( verbose_proxy_logger.debug("SCIM GET USER request for user_id=%s", user_id) try: user = await _check_user_exists(user_id) - + # Convert to SCIM format scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user) return scim_user @@ -458,6 +465,7 @@ async def get_user( except Exception as e: raise handle_exception_on_proxy(e) + @scim_router.post( "/Users", response_model=SCIMUser, @@ -471,11 +479,9 @@ async def create_user( Create a user according to SCIM v2 protocol """ try: - verbose_proxy_logger.debug( - "SCIM CREATE USER request: %s", user.model_dump() - ) + verbose_proxy_logger.debug("SCIM CREATE USER request: %s", user.model_dump()) prisma_client = await _get_prisma_client_or_raise_exception() - + # Extract data from SCIM user user_data = _extract_scim_user_data(user) @@ -487,20 +493,24 @@ async def create_user( if existing_user: raise HTTPException( status_code=409, - detail={"error": f"User already exists with username: {user.userName}"}, + detail={ + "error": f"User already exists with username: {user.userName}" + }, ) # Create user in database user_id = user.userName or str(uuid.uuid4()) - metadata = _build_scim_metadata(user_data["given_name"], user_data["family_name"]) + metadata = _build_scim_metadata( + user_data["given_name"], user_data["family_name"] + ) default_role: Optional[ Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] ] = LitellmUserRoles.INTERNAL_USER_VIEW_ONLY if litellm.default_internal_user_params: default_role = litellm.default_internal_user_params.get("user_role") @@ -517,22 +527,23 @@ async def create_user( # Check if user with email already exists and update if found existing_user_scim = await UserProvisionerHelpers.handle_existing_user_by_email( - prisma_client=prisma_client, - new_user_request=new_user_request + prisma_client=prisma_client, new_user_request=new_user_request ) - + if existing_user_scim: return existing_user_scim created_user = await new_user( data=new_user_request, ) - + scim_user = await ScimTransformations.transform_litellm_user_to_scim_user( user=created_user ) return scim_user - except HTTPException as e: # allow exceptions like SCIMUserAlreadyExists to be raised + except ( + HTTPException + ) as e: # allow exceptions like SCIMUserAlreadyExists to be raised raise e except Exception as e: raise handle_exception_on_proxy(e) @@ -564,18 +575,16 @@ async def update_user( # Extract data from SCIM user user_data = _extract_scim_user_data(user) - # Build metadata with SCIM data + # Build metadata with SCIM data metadata = _build_scim_metadata( - user_data["given_name"], - user_data["family_name"], - user_data["active"] + user_data["given_name"], user_data["family_name"], user_data["active"] ) # Handle team membership changes await _handle_team_membership_changes( user_id=user_id, existing_teams=existing_user.teams or [], - new_teams=user_data["teams"] + new_teams=user_data["teams"], ) # Update user with all new data (full replacement) @@ -590,6 +599,7 @@ async def update_user( # Serialize metadata to JSON string for Prisma to avoid GraphQL parsing issues if "metadata" in update_data and isinstance(update_data["metadata"], dict): from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + update_data["metadata"] = safe_dumps(update_data["metadata"]) updated_user = await prisma_client.db.litellm_usertable.update( @@ -598,8 +608,10 @@ async def update_user( ) # Convert back to SCIM format - scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(updated_user) - + scim_user = await ScimTransformations.transform_litellm_user_to_scim_user( + updated_user + ) + return scim_user except Exception as e: @@ -617,9 +629,7 @@ async def delete_user( """ Delete a user according to SCIM v2 protocol """ - verbose_proxy_logger.debug( - "SCIM DELETE USER request for user_id=%s", user_id - ) + verbose_proxy_logger.debug("SCIM DELETE USER request for user_id=%s", user_id) try: prisma_client = await _get_prisma_client_or_raise_exception() existing_user = await _check_user_exists(user_id) @@ -668,7 +678,9 @@ def _extract_group_values(value: Any) -> List[str]: return group_values -def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None: +def _handle_displayname_update( + op_type: str, value: Any, update_data: Dict[str, Any] +) -> None: """Handle displayname updates.""" if op_type == "remove": update_data["user_alias"] = None @@ -676,7 +688,9 @@ def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, update_data["user_alias"] = str(value) -def _handle_externalid_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None: +def _handle_externalid_update( + op_type: str, value: Any, update_data: Dict[str, Any] +) -> None: """Handle externalid updates.""" if op_type == "remove": update_data["sso_user_id"] = None @@ -697,7 +711,9 @@ def _handle_active_update(op_type: str, value: Any, metadata: Dict[str, Any]) -> metadata["scim_active"] = bool_val -def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict[str, Any]) -> None: +def _handle_name_update( + path: str, op_type: str, value: Any, scim_metadata: Dict[str, Any] +) -> None: """Handle name field updates (givenName, familyName).""" if path == "name.givenname": if op_type == "remove": @@ -711,7 +727,9 @@ def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict scim_metadata["familyName"] = str(value) -def _handle_group_operations(op_type: str, value: Any, teams_set: Set[str]) -> Optional[Set[str]]: +def _handle_group_operations( + op_type: str, value: Any, teams_set: Set[str] +) -> Optional[Set[str]]: """Handle group/team membership operations.""" group_values = _extract_group_values(value) if op_type == "replace": @@ -724,7 +742,9 @@ def _handle_group_operations(op_type: str, value: Any, teams_set: Set[str]) -> O return None -def _handle_generic_metadata(path: str, op_type: str, value: Any, metadata: Dict[str, Any]) -> None: +def _handle_generic_metadata( + path: str, op_type: str, value: Any, metadata: Dict[str, Any] +) -> None: """Handle generic metadata operations for unknown paths.""" if op_type == "remove": metadata.pop(path, None) @@ -769,6 +789,7 @@ def _apply_patch_ops( update_data["metadata"] = metadata return update_data, final_team_set + async def patch_team_membership( user_id: str, teams_ids_to_add_user_to: List[str], @@ -778,29 +799,35 @@ async def patch_team_membership( Add or remove user from teams """ for _team_id in teams_ids_to_add_user_to: - try: - await team_member_add( - data=TeamMemberAddRequest( - team_id=_team_id, - member=Member(user_id=user_id, role="user"), - ), - user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), - ) - except Exception as e: - verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}") + try: + await team_member_add( + data=TeamMemberAddRequest( + team_id=_team_id, + member=Member(user_id=user_id, role="user"), + ), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN + ), + ) + except Exception as e: + verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}") for _team_id in teams_ids_to_remove_user_from: try: await team_member_delete( data=TeamMemberDeleteRequest(team_id=_team_id, user_id=user_id), - user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN + ), ) except Exception as e: - verbose_proxy_logger.exception(f"Error removing user from team {_team_id}: {e}") - + verbose_proxy_logger.exception( + f"Error removing user from team {_team_id}: {e}" + ) return True + @scim_router.patch( "/Users/{user_id}", response_model=SCIMUser, @@ -833,7 +860,7 @@ async def patch_user( await _handle_team_membership_changes( user_id=user_id, existing_teams=existing_user.teams or [], - new_teams=list(final_team_set) + new_teams=list(final_team_set), ) update_data["teams"] = list(final_team_set) @@ -841,6 +868,7 @@ async def patch_user( # Serialize metadata to JSON string for Prisma to avoid GraphQL parsing issues if "metadata" in update_data and isinstance(update_data["metadata"], dict): from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + update_data["metadata"] = safe_dumps(update_data["metadata"]) updated_user = await prisma_client.db.litellm_usertable.update( @@ -848,7 +876,9 @@ async def patch_user( data=update_data, ) - scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(updated_user) + scim_user = await ScimTransformations.transform_litellm_user_to_scim_user( + updated_user + ) return scim_user @@ -947,9 +977,7 @@ async def get_group( """ Get a single group by ID according to SCIM v2 protocol """ - verbose_proxy_logger.debug( - "SCIM GET GROUP request for group_id=%s", group_id - ) + verbose_proxy_logger.debug("SCIM GET GROUP request for group_id=%s", group_id) try: team = await _check_team_exists(group_id) @@ -981,9 +1009,9 @@ async def create_group( ) try: prisma_client = await _get_prisma_client_or_raise_exception() - + # Generate ID if not provided - team_id = group.id or str(uuid.uuid4()) + team_id = group.id or group.externalId or str(uuid.uuid4()) # Check if team already exists existing_team = await prisma_client.db.litellm_teamtable.find_unique( @@ -998,7 +1026,10 @@ async def create_group( # Extract and process group members (creating users that don't exist) member_result = await _extract_group_member_ids(group) - members_with_roles = [Member(user_id=member_id, role="user") for member_id in member_result.all_member_ids] + members_with_roles = [ + Member(user_id=member_id, role="user") + for member_id in member_result.all_member_ids + ] # Create team in database created_team = await new_team( @@ -1043,13 +1074,17 @@ async def update_group( # Extract and process group members (creating users that don't exist) member_result = await _extract_group_member_ids(group) - verbose_proxy_logger.debug(f"SCIM PUT GROUP all_member_ids: {member_result.all_member_ids}") - verbose_proxy_logger.debug(f"SCIM PUT GROUP created_users: {len(member_result.created_users)}") + verbose_proxy_logger.debug( + f"SCIM PUT GROUP all_member_ids: {member_result.all_member_ids}" + ) + verbose_proxy_logger.debug( + f"SCIM PUT GROUP created_users: {len(member_result.created_users)}" + ) # Prepare update data existing_metadata = existing_team.metadata if existing_team.metadata else {} updated_metadata = {**existing_metadata, "scim_data": group.model_dump()} - + update_data = { "team_alias": group.displayName, "metadata": safe_dumps(updated_metadata), @@ -1066,7 +1101,7 @@ async def update_group( verbose_proxy_logger.debug(f"SCIM PUT GROUP current_members: {current_members}") final_members = set(member_result.all_member_ids) verbose_proxy_logger.debug(f"SCIM PUT GROUP final_members: {final_members}") - + await _handle_group_membership_changes( group_id=group_id, current_members=current_members, @@ -1094,9 +1129,7 @@ async def delete_group( """ Delete a group according to SCIM v2 protocol """ - verbose_proxy_logger.debug( - "SCIM DELETE GROUP request for group_id=%s", group_id - ) + verbose_proxy_logger.debug("SCIM DELETE GROUP request for group_id=%s", group_id) try: prisma_client = await _get_prisma_client_or_raise_exception() existing_team = await _check_team_exists(group_id) @@ -1124,21 +1157,19 @@ async def delete_group( async def _process_group_patch_operations( - patch_ops: SCIMPatchOp, - existing_team, - prisma_client + patch_ops: SCIMPatchOp, existing_team, prisma_client ) -> Tuple[Dict[str, Any], Set[str]]: """Process patch operations for a group and return update data and final members.""" update_data: Dict[str, Any] = {} - + # Create a fresh copy of existing metadata to avoid Prisma issues existing_metadata = existing_team.metadata or {} metadata = dict(existing_metadata) if existing_metadata else {} - + # Track member changes current_members = set(existing_team.members or []) final_members = current_members.copy() - + # Process each patch operation for op in patch_ops.Operations: path = (op.path or "").lower() @@ -1169,14 +1200,13 @@ async def _process_group_patch_operations( else: # Create the user if they don't exist using our helper created_user = await _create_user_if_not_exists( - user_id=member_id, - created_via="scim_group_patch" + user_id=member_id, created_via="scim_group_patch" ) - + if created_user: valid_members.append(member_id) # If creation failed, user is skipped (logged in helper) - + if op_type == "replace": final_members = set(valid_members) elif op_type == "add": @@ -1194,21 +1224,18 @@ async def _process_group_patch_operations( # Include metadata in update data if it exists if metadata: update_data["metadata"] = metadata - + return update_data, final_members async def _apply_group_patch_updates( - group_id: str, - update_data: Dict[str, Any], - final_members: Set[str], - prisma_client + group_id: str, update_data: Dict[str, Any], final_members: Set[str], prisma_client ): """Apply patch updates to the group in the database.""" # Serialize metadata if present if "metadata" in update_data and isinstance(update_data["metadata"], dict): update_data["metadata"] = safe_dumps(update_data["metadata"]) - + # Update members list update_data["members"] = list(final_members) @@ -1217,22 +1244,20 @@ async def _apply_group_patch_updates( where={"team_id": group_id}, data=update_data, ) - + return updated_team async def _handle_group_membership_changes( - group_id: str, - current_members: Set[str], - final_members: Set[str] + group_id: str, current_members: Set[str], final_members: Set[str] ): """Handle adding/removing members from the group.""" members_to_add = final_members - current_members members_to_remove = current_members - final_members - + verbose_proxy_logger.debug(f"members_to_add: {members_to_add}") verbose_proxy_logger.debug(f"members_to_remove: {members_to_remove}") - + # Use existing helper functions for team membership changes for member_id in members_to_add: await patch_team_membership( @@ -1276,7 +1301,7 @@ async def patch_group( update_data, final_members = await _process_group_patch_operations( patch_ops, existing_team, prisma_client ) - + # Track current members for comparison current_members = set(await _get_team_member_user_ids_from_team(existing_team)) @@ -1286,9 +1311,7 @@ async def patch_group( ) # Handle user-team relationship changes - await _handle_group_membership_changes( - group_id, current_members, final_members - ) + await _handle_group_membership_changes(group_id, current_members, final_members) # Convert to SCIM format and return scim_group = await ScimTransformations.transform_litellm_team_to_scim_group( From 6c4436215c54085ea2f3cb93d092bb6c1ebdc8ea Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 7 Oct 2025 17:19:18 -0700 Subject: [PATCH 02/39] feat(pass_through_endpoints.py): initial commit trying to delete passthrough endpoints successfully --- .../pass_through_endpoints.py | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 227ea4db9ff..5f3a49f307a 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1661,6 +1661,11 @@ class InitPassThroughEndpointHelpers: "Removed pass-through route from registry: %s", key ) + @staticmethod + def clear_all_pass_through_routes(): + """Clear all pass-through routes from the registry""" + _registered_pass_through_routes.clear() + @staticmethod def is_registered_pass_through_route(route: str) -> bool: """ @@ -1695,10 +1700,22 @@ class InitPassThroughEndpointHelpers: return False +def _get_combined_pass_through_endpoints( + pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], + config_pass_through_endpoints: List[Dict], +): + """Get combined pass-through endpoints from db + config""" + return pass_through_endpoints + config_pass_through_endpoints + + async def initialize_pass_through_endpoints( pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], ): """ + 1. Create a global list of pass-through endpoints (db + config) + 2. Clear all existing pass-through endpoints from the FastAPI app routes + 3. Add new endpoints to the in-memory registry + Initialize a list of pass-through endpoints by adding them to the FastAPI app routes Args: @@ -1711,9 +1728,22 @@ async def initialize_pass_through_endpoints( verbose_proxy_logger.debug("initializing pass through endpoints") from litellm.proxy._types import CommonProxyErrors, LiteLLMRoutes - from litellm.proxy.proxy_server import app, premium_user + from litellm.proxy.proxy_server import app, general_settings, premium_user - for endpoint in pass_through_endpoints: + ## get combined pass-through endpoints from db + config + config_pass_through_endpoints = general_settings.get("pass_through_endpoints") + combined_pass_through_endpoints: List[Union[Dict, PassThroughGenericEndpoint]] + if config_pass_through_endpoints is not None: + combined_pass_through_endpoints = _get_combined_pass_through_endpoints( # type: ignore + pass_through_endpoints, config_pass_through_endpoints + ) + else: + combined_pass_through_endpoints = pass_through_endpoints # type: ignore + + ## clear all existing pass-through endpoints from the FastAPI app routes + InitPassThroughEndpointHelpers.clear_all_pass_through_routes() + + for endpoint in combined_pass_through_endpoints: if isinstance(endpoint, PassThroughGenericEndpoint): endpoint = endpoint.model_dump() From b4fc7d3cfad04e40e257be8ceb193bf6074f636e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 7 Oct 2025 17:58:15 -0700 Subject: [PATCH 03/39] feat(pass_through_endpoints.py): raise 404 if passthrough endpoint has been deleted by the user Ensures passthrough endpoint deletion works as expected (single-instance) --- .../pass_through_endpoints.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 5f3a49f307a..8542f87327f 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -974,6 +974,11 @@ def create_pass_through_route( ] = None, # if pass-through endpoint is a streaming request subpath: str = "", # captures sub-paths when include_subpath=True ): + + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + ) + # Construct the full target URL with subpath if needed full_target = ( HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( @@ -981,6 +986,14 @@ def create_pass_through_route( ) ) + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( + route=endpoint + ): + raise HTTPException( + status_code=404, + detail=f"Pass-through endpoint {target} not found. This could have been deleted or not yet added to the proxy.", + ) + return await pass_through_request( # type: ignore request=request, target=full_target, From 032a213b9bf388fcbed4624e023155272a1c1b2d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 7 Oct 2025 18:04:44 -0700 Subject: [PATCH 04/39] feat(pass_through_endpoint.py): cleanup error message + validate it works across multiple instances --- litellm/proxy/pass_through_endpoints/pass_through_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 8542f87327f..b1877cd20ea 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -991,7 +991,7 @@ def create_pass_through_route( ): raise HTTPException( status_code=404, - detail=f"Pass-through endpoint {target} not found. This could have been deleted or not yet added to the proxy.", + detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", ) return await pass_through_request( # type: ignore From c671dceb24d9f411636072da1c14f131400552bc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 7 Oct 2025 19:03:32 -0700 Subject: [PATCH 05/39] feat(pass_through_endpoints.py): have updates not require pod restarts ensures db updates work on live passthrough endpoints without requiring pod restarts --- .../pass_through_endpoints.py | 83 ++++++++++++++++--- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b1877cd20ea..1251e85468d 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -66,7 +66,7 @@ router = APIRouter() pass_through_endpoint_logging = PassThroughEndpointLogging() # Global registry to track registered pass-through routes and prevent memory leaks -_registered_pass_through_routes: Dict[str, Dict[str, str]] = {} +_registered_pass_through_routes: Dict[str, Dict[str, Union[str, Dict[str, Any]]]] = {} def get_response_body(response: httpx.Response) -> Optional[dict]: @@ -979,13 +979,6 @@ def create_pass_through_route( InitPassThroughEndpointHelpers, ) - # Construct the full target URL with subpath if needed - full_target = ( - HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( - base_target=target, subpath=subpath, include_subpath=include_subpath - ) - ) - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( route=endpoint ): @@ -994,17 +987,47 @@ def create_pass_through_route( detail=f"Pass-through endpoint {endpoint} not found. This could have been deleted or not yet added to the proxy.", ) + passthrough_params = ( + InitPassThroughEndpointHelpers.get_registered_pass_through_route( + route=endpoint + ) + ) + target_params = { + "target": target, + "custom_headers": custom_headers, + "forward_headers": _forward_headers, + "merge_query_params": _merge_query_params, + "cost_per_request": cost_per_request, + } + + if passthrough_params is not None: + target_params.update(passthrough_params.get("passthrough_params", {})) + + # Construct the full target URL with subpath if needed + full_target = ( + HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( + base_target=target_params.get("target", target), + subpath=subpath, + include_subpath=include_subpath, + ) + ) + return await pass_through_request( # type: ignore request=request, target=full_target, - custom_headers=custom_headers or {}, + custom_headers=target_params.get("custom_headers", custom_headers) + or {}, user_api_key_dict=user_api_key_dict, - forward_headers=_forward_headers, - merge_query_params=_merge_query_params, + forward_headers=target_params.get("forward_headers", _forward_headers), + merge_query_params=target_params.get( + "merge_query_params", _merge_query_params + ), query_params=query_params, stream=stream, custom_body=custom_body, - cost_per_request=cost_per_request, + cost_per_request=target_params.get( + "cost_per_request", cost_per_request + ), custom_llm_provider=custom_llm_provider, ) @@ -1605,6 +1628,14 @@ class InitPassThroughEndpointHelpers: "endpoint_id": endpoint_id, "path": path, "type": "exact", + "passthrough_params": { + "target": target, + "custom_headers": custom_headers, + "forward_headers": forward_headers, + "merge_query_params": merge_query_params, + "dependencies": dependencies, + "cost_per_request": cost_per_request, + }, } @staticmethod @@ -1658,6 +1689,14 @@ class InitPassThroughEndpointHelpers: "endpoint_id": endpoint_id, "path": path, "type": "subpath", + "passthrough_params": { + "target": target, + "custom_headers": custom_headers, + "forward_headers": forward_headers, + "merge_query_params": merge_query_params, + "dependencies": dependencies, + "cost_per_request": cost_per_request, + }, } @staticmethod @@ -1712,6 +1751,25 @@ class InitPassThroughEndpointHelpers: return False + @staticmethod + def get_registered_pass_through_route(route: str) -> Optional[Dict[str, Any]]: + """Get passthrough params for a given route""" + for key in _registered_pass_through_routes.keys(): + parts = key.split(":", 2) # Split into [endpoint_id, type, path] + if len(parts) == 3: + route_type = parts[1] + registered_path = parts[2] + + if route_type == "exact" and route == registered_path: + return _registered_pass_through_routes[key] + elif route_type == "subpath": + if route == registered_path or route.startswith( + registered_path + "/" + ): + return _registered_pass_through_routes[key] + + return None + def _get_combined_pass_through_endpoints( pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]], @@ -1973,6 +2031,7 @@ async def update_pass_through_endpoints( field_value=pass_through_endpoint_data, config_type="general_settings", ) + await update_config_general_settings( data=updated_data, user_api_key_dict=user_api_key_dict ) From b90ff30b2af2d2e329e88586c3c8bc0f6ec7c80f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 8 Oct 2025 19:37:38 -0700 Subject: [PATCH 06/39] fix: fix linting errors --- .../pass_through_endpoints.py | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 1251e85468d..3bf5f0e5e44 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -5,7 +5,7 @@ import json import traceback from base64 import b64encode from datetime import datetime -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urlparse import httpx @@ -1003,31 +1003,44 @@ def create_pass_through_route( if passthrough_params is not None: target_params.update(passthrough_params.get("passthrough_params", {})) + # Extract and cast parameters with proper types + param_target = target_params.get("target") or target + param_custom_headers = target_params.get("custom_headers", custom_headers) + param_forward_headers = target_params.get( + "forward_headers", _forward_headers + ) + param_merge_query_params = target_params.get( + "merge_query_params", _merge_query_params + ) + param_cost_per_request = target_params.get( + "cost_per_request", cost_per_request + ) + # Construct the full target URL with subpath if needed full_target = ( HttpPassThroughEndpointHelpers.construct_target_url_with_subpath( - base_target=target_params.get("target", target), + base_target=cast(str, param_target), subpath=subpath, include_subpath=include_subpath, ) ) + # Ensure custom_headers is a dict + headers_dict = ( + param_custom_headers if isinstance(param_custom_headers, dict) else {} + ) + return await pass_through_request( # type: ignore request=request, target=full_target, - custom_headers=target_params.get("custom_headers", custom_headers) - or {}, + custom_headers=headers_dict, user_api_key_dict=user_api_key_dict, - forward_headers=target_params.get("forward_headers", _forward_headers), - merge_query_params=target_params.get( - "merge_query_params", _merge_query_params - ), + forward_headers=cast(Optional[bool], param_forward_headers), + merge_query_params=cast(Optional[bool], param_merge_query_params), query_params=query_params, stream=stream, custom_body=custom_body, - cost_per_request=target_params.get( - "cost_per_request", cost_per_request - ), + cost_per_request=cast(Optional[float], param_cost_per_request), custom_llm_provider=custom_llm_provider, ) From d956f7417f099871868fecda8bd52d4a5f1cc9a4 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 14:42:04 -0700 Subject: [PATCH 07/39] fix(client.py): fix rest api tool call --- litellm/experimental_mcp_client/client.py | 125 ++++++++++++++++-- .../mcp_server/mcp_server_manager.py | 22 +++ 2 files changed, 138 insertions(+), 9 deletions(-) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index b10ddc9e812..6aa671a5011 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -86,8 +86,15 @@ class MCPClient: async def connect(self): """Initialize the transport and session.""" if self._session: + verbose_logger.debug( + f"MCP client already connected to {self.server_url or 'stdio'}" + ) return # Already connected + verbose_logger.info( + f"MCP client connecting to {self.server_url or 'stdio'} via {self.transport_type}" + ) + try: if self.transport_type == MCPTransport.stdio: # For stdio transport, use stdio_client with command-line parameters @@ -107,6 +114,9 @@ class MCPClient: ) self._session = await self._session_ctx.__aenter__() await self._session.initialize() + verbose_logger.info( + f"MCP client successfully connected via stdio: {self.stdio_config.get('command', '')}" + ) elif self.transport_type == MCPTransport.sse: headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() @@ -122,6 +132,9 @@ class MCPClient: ) self._session = await self._session_ctx.__aenter__() await self._session.initialize() + verbose_logger.info( + f"MCP client successfully connected via SSE to {self.server_url}" + ) else: # http headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() @@ -140,6 +153,9 @@ class MCPClient: ) self._session = await self._session_ctx.__aenter__() await self._session.initialize() + verbose_logger.info( + f"MCP client successfully connected via HTTP to {self.server_url}" + ) except ValueError as e: # Re-raise ValueError exceptions (like missing stdio_config) verbose_logger.warning(f"MCP client connection failed: {str(e)}") @@ -159,7 +175,12 @@ class MCPClient: async def disconnect(self): """Clean up session and connections.""" + verbose_logger.info( + f"MCP client disconnecting from {self.server_url or 'stdio'}" + ) + if self._task and not self._task.done(): + verbose_logger.debug("MCP client cancelling background task") self._task.cancel() try: await self._task @@ -168,16 +189,24 @@ class MCPClient: if self._session: try: + verbose_logger.debug("MCP client closing session") await self._session_ctx.__aexit__(None, None, None) # type: ignore - except Exception: + except Exception as e: + verbose_logger.debug( + f"Error closing MCP session: {type(e).__name__}: {str(e)}" + ) pass self._session = None self._session_ctx = None if self._transport_ctx: try: + verbose_logger.debug("MCP client closing transport") await self._transport_ctx.__aexit__(None, None, None) - except Exception: + except Exception as e: + verbose_logger.debug( + f"Error closing MCP transport: {type(e).__name__}: {str(e)}" + ) pass self._transport_ctx = None self._transport = None @@ -261,25 +290,55 @@ class MCPClient: async def list_tools(self) -> List[MCPTool]: """List available tools from the server.""" + verbose_logger.debug( + f"MCP client listing tools from {self.server_url or 'stdio'}" + ) + if not self._session: + verbose_logger.debug("MCP client session not found, attempting to connect") try: await self.connect() except Exception as e: - verbose_logger.warning(f"MCP client connection failed: {str(e)}") + verbose_logger.error( + f"MCP client connection failed during list_tools: {type(e).__name__}: {str(e)}" + ) return [] if self._session is None: - verbose_logger.warning("MCP client session is not initialized") + verbose_logger.error( + "MCP client session is not initialized after connection attempt" + ) return [] try: result = await self._session.list_tools() + tool_count = len(result.tools) + tool_names = [tool.name for tool in result.tools] + verbose_logger.info( + f"MCP client listed {tool_count} tools from {self.server_url or 'stdio'}: {tool_names}" + ) return result.tools except asyncio.CancelledError: + verbose_logger.warning("MCP client list_tools was cancelled") await self.disconnect() raise except Exception as e: - verbose_logger.warning(f"MCP client list_tools failed: {str(e)}") + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client list_tools failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream during list_tools - " + "the MCP server may have crashed, disconnected, or timed out" + ) + await self.disconnect() # Return empty list instead of raising to allow graceful degradation return [] @@ -290,17 +349,28 @@ class MCPClient: """ Call an MCP Tool. """ + verbose_logger.info( + f"MCP client calling tool '{call_tool_request_params.name}' with arguments: {call_tool_request_params.arguments}" + ) + if not self._session: + verbose_logger.warning( + "MCP client session not found, attempting to connect" + ) try: await self.connect() except Exception as e: - verbose_logger.warning(f"MCP client connection failed: {str(e)}") + verbose_logger.error( + f"MCP client connection failed before tool call: {type(e).__name__}: {str(e)}" + ) return MCPCallToolResult( content=[TextContent(type="text", text=f"{str(e)}")], isError=True ) if self._session is None: - verbose_logger.warning("MCP client session is not initialized") + verbose_logger.error( + "MCP client session is not initialized after connection attempt" + ) return MCPCallToolResult( content=[ TextContent( @@ -310,22 +380,59 @@ class MCPClient: isError=True, ) + # Check session and transport state before calling tool + verbose_logger.debug( + f"MCP client state before tool call - " + f"session: {'active' if self._session else 'none'}, " + f"transport: {'active' if self._transport else 'none'}, " + f"session_ctx: {'active' if self._session_ctx else 'none'}, " + f"transport_ctx: {'active' if self._transport_ctx else 'none'}" + ) + try: + verbose_logger.debug("MCP client sending tool call to session") tool_result = await self._session.call_tool( name=call_tool_request_params.name, arguments=call_tool_request_params.arguments, ) + verbose_logger.info( + f"MCP client tool call '{call_tool_request_params.name}' completed successfully" + ) return tool_result except asyncio.CancelledError: + verbose_logger.warning("MCP client tool call was cancelled") await self.disconnect() raise except Exception as e: - verbose_logger.warning(f"MCP client call_tool failed: {str(e)}") + import traceback + + error_trace = traceback.format_exc() + verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}") + + # Log detailed error information + error_type = type(e).__name__ + verbose_logger.error( + f"MCP client call_tool failed - " + f"Error Type: {error_type}, " + f"Error: {str(e)}, " + f"Tool: {call_tool_request_params.name}, " + f"Server: {self.server_url or 'stdio'}, " + f"Transport: {self.transport_type}" + ) + + # Check if it's a stream/connection error + if "BrokenResourceError" in error_type or "Broken" in error_type: + verbose_logger.error( + "MCP client detected broken connection/stream - " + "the MCP server may have crashed, disconnected, or timed out. " + "Session and transport will be disconnected." + ) + await self.disconnect() # Return a default error result instead of raising return MCPCallToolResult( content=[ - TextContent(type="text", text=f"{str(e)}") + TextContent(type="text", text=f"{error_type}: {str(e)}") ], # Empty content for error case isError=True, ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index ffdcfe8679f..9b7ccf7b41c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1166,6 +1166,28 @@ class MCPServerManager: ) tasks.append(asyncio.create_task(client.call_tool(call_tool_params))) + # IMPORTANT: Must await tasks INSIDE the context manager to keep connection alive + try: + mcp_responses = await asyncio.gather(*tasks) + except ( + BlockedPiiEntityError, + GuardrailRaisedException, + HTTPException, + ) as e: + # Re-raise guardrail exceptions to properly fail the MCP call + verbose_logger.error( + f"Guardrail blocked MCP tool call during result check: {str(e)}" + ) + raise e + + # If proxy_logging_obj is None, the tool call result is at index 0 + # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task) + result_index = 1 if proxy_logging_obj else 0 + result = mcp_responses[result_index] + + return cast(CallToolResult, result) + + # For OpenAPI tools, await outside the client context try: mcp_responses = await asyncio.gather(*tasks) From ace862189c12c331cdcf1bf4b12258caf0eabfeb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 14:52:46 -0700 Subject: [PATCH 08/39] test(test_mcp_server_manager.py): add unit testing --- .gitignore | 2 + .../index.html} | 0 .../proxy/_experimental/out/guardrails.html | 1 - .../out/{logs.html => logs/index.html} | 0 .../{model-hub.html => model-hub/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../proxy/_experimental/out/onboarding.html | 1 - .../index.html} | 0 .../out/{teams.html => teams/index.html} | 0 .../{test-key.html => test-key/index.html} | 0 .../out/{usage.html => usage/index.html} | 0 .../out/{users.html => users/index.html} | 0 .../index.html} | 0 .../mcp_server/test_mcp_server_manager.py | 98 +++++++++++++++++++ 15 files changed, 100 insertions(+), 2 deletions(-) rename litellm/proxy/_experimental/out/{api-reference.html => api-reference/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/guardrails.html rename litellm/proxy/_experimental/out/{logs.html => logs/index.html} (100%) rename litellm/proxy/_experimental/out/{model-hub.html => model-hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) rename litellm/proxy/_experimental/out/{models-and-endpoints.html => models-and-endpoints/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html rename litellm/proxy/_experimental/out/{organizations.html => organizations/index.html} (100%) rename litellm/proxy/_experimental/out/{teams.html => teams/index.html} (100%) rename litellm/proxy/_experimental/out/{test-key.html => test-key/index.html} (100%) rename litellm/proxy/_experimental/out/{usage.html => usage/index.html} (100%) rename litellm/proxy/_experimental/out/{users.html => users/index.html} (100%) rename litellm/proxy/_experimental/out/{virtual-keys.html => virtual-keys/index.html} (100%) diff --git a/.gitignore b/.gitignore index c2ac5137cbe..e1045032d46 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,5 @@ litellm_config.yaml .vscode/launch.json litellm/proxy/to_delete_loadtest_work/* update_model_cost_map.py +tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +litellm/proxy/_experimental/out/guardrails/index.html diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails.html deleted file mode 100644 index fb918264d26..00000000000 --- a/litellm/proxy/_experimental/out/guardrails.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 5df786e1f14..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 0476f290a09..bfb2f4bf195 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1222,6 +1222,104 @@ class TestMCPServerManager: "Contact proxy admin to allow this tool" in exc_info.value.detail["error"] ) + @pytest.mark.asyncio + async def test_call_tool_without_broken_pipe_error(self): + """ + Test that call_tool properly uses async context manager to avoid broken pipe errors. + This test ensures that tasks are awaited INSIDE the context manager, keeping the connection alive. + """ + from unittest.mock import AsyncMock, MagicMock, patch + + from mcp.types import CallToolResult + + manager = MCPServerManager() + + # Create a test server + server = MCPServer( + server_id="test-server", + name="test-server", + transport=MCPTransport.http, + url="http://test-server.com", + ) + + # Register the server and map a tool to it + manager.registry = {"test-server": server} + manager.tool_name_to_mcp_server_name_mapping["test_tool"] = "test-server" + + # Create mock client that tracks context manager usage + mock_client = MagicMock() + context_entered = False + context_exited = False + call_tool_called_inside_context = False + + async def mock_aenter(self): + nonlocal context_entered + context_entered = True + return self + + async def mock_aexit(self, exc_type, exc_val, exc_tb): + nonlocal context_exited + context_exited = True + # Verify that call_tool was called before context exit + assert ( + call_tool_called_inside_context + ), "call_tool must be awaited inside context manager" + return False + + async def mock_call_tool(params): + nonlocal call_tool_called_inside_context + # Verify we're inside the context when this is called + assert context_entered, "call_tool called outside context manager" + assert not context_exited, "call_tool called after context exit" + call_tool_called_inside_context = True + + # Return a mock CallToolResult + result = MagicMock(spec=CallToolResult) + result.content = [{"type": "text", "text": "Tool executed successfully"}] + result.isError = False + return result + + mock_client.__aenter__ = mock_aenter + mock_client.__aexit__ = mock_aexit + mock_client.call_tool = mock_call_tool + + # Mock _create_mcp_client to return our mock client + manager._create_mcp_client = MagicMock(return_value=mock_client) + + # Mock user auth with no restrictions + user_api_key_auth = MagicMock() + user_api_key_auth.object_permission = None + user_api_key_auth.object_permission_id = None + + # Mock proxy logging + proxy_logging_obj = MagicMock() + proxy_logging_obj._create_mcp_request_object_from_kwargs = MagicMock( + return_value={} + ) + proxy_logging_obj._convert_mcp_to_llm_format = MagicMock(return_value={}) + proxy_logging_obj.pre_call_hook = AsyncMock(return_value={}) + proxy_logging_obj.during_call_hook = AsyncMock(return_value=None) + + # Call the tool + result = await manager.call_tool( + name="test_tool", + arguments={"param": "value"}, + user_api_key_auth=user_api_key_auth, + proxy_logging_obj=proxy_logging_obj, + ) + + # Verify the result + assert result is not None + assert result.isError is False + assert len(result.content) > 0 + + # Verify context manager was used properly + assert context_entered, "Context manager __aenter__ was not called" + assert context_exited, "Context manager __aexit__ was not called" + assert ( + call_tool_called_inside_context + ), "call_tool was not awaited inside context" + if __name__ == "__main__": pytest.main([__file__]) From 732495fb3a265095d930b0872304f9a1ce1fefde Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 15:16:10 -0700 Subject: [PATCH 09/39] fix(route_checks.py): handle route prefixes --- litellm/proxy/_new_secret_config.yaml | 8 ------ litellm/proxy/auth/route_checks.py | 41 +++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 95c6869b95b..68297d4a5fe 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -16,14 +16,6 @@ model_list: api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" api_key: dummy -mcp_servers: - my_api_mcp: - url: "http://0.0.0.0:8090" - spec_path: "/Users/krrishdholakia/Documents/temp_py_folder/example_openapi.json" - auth_type: none - allowed_tools: ["getpetbyid", "my_api_mcp-findpetsbystatus"] - - litellm_settings: callbacks: ["prometheus"] custom_prometheus_metadata_labels: ["metadata.initiative", "metadata.business-unit"] diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 39f11e64bb7..c1f4966ac37 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -56,19 +56,25 @@ class RouteChecks: if route in valid_token.allowed_routes: return True + # check if any allowed_route is a prefix of the actual route + # e.g., allowed_route="/fake-openai-proxy-6" matches route="/fake-openai-proxy-6/v1/chat/completions" + for allowed_route in valid_token.allowed_routes: + if route.startswith(allowed_route + "/") or route == allowed_route: + return True + ## check if 'allowed_route' is a field name in LiteLLMRoutes if any( allowed_route in LiteLLMRoutes._member_names_ for allowed_route in valid_token.allowed_routes ): for allowed_route in valid_token.allowed_routes: - if allowed_route in LiteLLMRoutes._member_names_: + if allowed_route in LiteLLMRoutes._member_names_: if RouteChecks.check_route_access( route=route, allowed_routes=LiteLLMRoutes._member_map_[allowed_route].value, ): return True - + ################################################ # For llm_api_routes, also check registered pass-through endpoints ################################################ @@ -76,7 +82,10 @@ class RouteChecks: from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, ) - if InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route): + + if InitPassThroughEndpointHelpers.is_registered_pass_through_route( + route=route + ): return True # check if wildcard pattern is allowed @@ -195,6 +204,32 @@ class RouteChecks: pass elif route.startswith("/v1/mcp/") or route.startswith("/mcp-rest/"): pass # authN/authZ handled by api itself + elif valid_token.allowed_routes is not None: + # check if route is in allowed_routes (exact match or prefix match) + route_allowed = False + if route in valid_token.allowed_routes: + route_allowed = True + else: + # check if any allowed_route is a prefix of the actual route + # e.g., allowed_route="/fake-openai-proxy-6" matches route="/fake-openai-proxy-6/v1/chat/completions" + for allowed_route in valid_token.allowed_routes: + if route.startswith(allowed_route + "/") or route == allowed_route: + route_allowed = True + break + + if route_allowed: + pass + else: + user_role = "unknown" + user_id = "unknown" + if user_obj is not None: + user_role = user_obj.user_role or "unknown" + user_id = user_obj.user_id or "unknown" + + masked_user_id = RouteChecks._mask_user_id(user_id) + raise Exception( + f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" + ) else: user_role = "unknown" user_id = "unknown" From ecc391be6b937dd0e45adc3745fbbcc270d5ff9b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 15:35:31 -0700 Subject: [PATCH 10/39] fix(key_management_endpoints.py): support new `allowed_passthrough_routes` param on key generate allows admin to give devs access to specific passthrough endpoints Work for LIT-1182 --- litellm/proxy/_types.py | 4 +++- .../proxy/management_endpoints/key_management_endpoints.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 096bbc029b5..babe3122fee 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -775,6 +775,7 @@ class KeyRequestBase(GenerateRequestBase): tags: Optional[List[str]] = None enforced_params: Optional[List[str]] = None allowed_routes: Optional[list] = [] + allowed_passthrough_routes: Optional[list] = [] rpm_limit_type: Optional[ Literal["guaranteed_throughput", "best_effort_throughput"] ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm @@ -1442,7 +1443,7 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase): "1234567890": ["tool_name_1", "tool_name_2"] } """ - + vector_stores: Optional[List[str]] = [] @@ -3105,6 +3106,7 @@ LiteLLM_ManagementEndpoint_MetadataFields = [ "enforced_params", "temp_budget_increase", "temp_budget_expiry", + "allowed_passthrough_routes", ] LiteLLM_ManagementEndpoint_MetadataFields_Premium = [ diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 3226ea01798..772cb42d3e6 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -47,9 +47,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( _add_model_to_db, ) from litellm.proxy.management_helpers.object_permission_utils import ( + _set_object_permission, attach_object_permission_to_dict, handle_update_object_permission_common, - _set_object_permission, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, @@ -563,6 +563,7 @@ async def _common_key_generation_helper( # noqa: PLR0915 data_json = data.model_dump(exclude_unset=True, exclude_none=True) # type: ignore data_json = handle_key_type(data, data_json) + # if we get max_budget passed to /key/generate, then use it as key_max_budget. Since generate_key_helper_fn is used to make new users if "max_budget" in data_json: data_json["key_max_budget"] = data_json.pop("max_budget", None) @@ -838,6 +839,7 @@ async def generate_key_fn( - enforced_params: Optional[List[str]] - List of enforced params for the key (Enterprise only). [Docs](https://docs.litellm.ai/docs/proxy/enterprise#enforce-required-params-for-llm-requests) - prompts: Optional[List[str]] - List of prompts that the key is allowed to use. - allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"] + - allowed_passthrough_routes: Optional[list] - List of allowed pass through endpoints for the key. Store the actual endpoint or store a wildcard pattern for a set of endpoints. Example - ["/my-custom-endpoint"]. Use this instead of allowed_routes, if you just want to specify which pass through endpoints the key can access, without specifying the routes. If allowed_routes is specified, allowed_pass_through_endpoints is ignored. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "mcp_tool_permissions": {"server_id_1": ["tool1", "tool2"]}}. IF null or {} then no object permission. - key_type: Optional[str] - Type of key that determines default allowed routes. Options: "llm_api" (can call LLM API routes), "management" (can call management routes), "read_only" (can only call info/read routes), "default" (uses default allowed routes). Defaults to "default". - prompts: Optional[List[str]] - List of allowed prompts for the key. If specified, the key will only be able to use these specific prompts. @@ -1114,8 +1116,6 @@ def prepare_metadata_fields( return non_default_values - - async def prepare_key_update_data( data: Union[UpdateKeyRequest, RegenerateKeyRequest], existing_key_row: LiteLLM_VerificationToken, @@ -2819,6 +2819,7 @@ async def list_keys( code=status.HTTP_500_INTERNAL_SERVER_ERROR, ) + @router.get( "/key/aliases", tags=["key management"], From 1d26397f17ea8fd0fa4a28cbff12fecf73994884 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 15:46:15 -0700 Subject: [PATCH 11/39] fix(route_checks.py): ensure allowed passthrough endpoint routes are respected --- litellm/proxy/auth/route_checks.py | 83 ++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index c1f4966ac37..19d152cbdc7 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -52,14 +52,11 @@ class RouteChecks: if len(valid_token.allowed_routes) == 0: return True - # explicit check for allowed routes - if route in valid_token.allowed_routes: - return True - - # check if any allowed_route is a prefix of the actual route - # e.g., allowed_route="/fake-openai-proxy-6" matches route="/fake-openai-proxy-6/v1/chat/completions" + # explicit check for allowed routes (exact match or prefix match) for allowed_route in valid_token.allowed_routes: - if route.startswith(allowed_route + "/") or route == allowed_route: + if RouteChecks._route_matches_allowed_route( + route=route, allowed_route=allowed_route + ): return True ## check if 'allowed_route' is a field name in LiteLLMRoutes @@ -204,18 +201,19 @@ class RouteChecks: pass elif route.startswith("/v1/mcp/") or route.startswith("/mcp-rest/"): pass # authN/authZ handled by api itself + elif RouteChecks.check_passthrough_route_access( + route=route, user_api_key_dict=valid_token + ): + pass elif valid_token.allowed_routes is not None: # check if route is in allowed_routes (exact match or prefix match) route_allowed = False - if route in valid_token.allowed_routes: - route_allowed = True - else: - # check if any allowed_route is a prefix of the actual route - # e.g., allowed_route="/fake-openai-proxy-6" matches route="/fake-openai-proxy-6/v1/chat/completions" - for allowed_route in valid_token.allowed_routes: - if route.startswith(allowed_route + "/") or route == allowed_route: - route_allowed = True - break + for allowed_route in valid_token.allowed_routes: + if RouteChecks._route_matches_allowed_route( + route=route, allowed_route=allowed_route + ): + route_allowed = True + break if route_allowed: pass @@ -382,6 +380,32 @@ class RouteChecks: # If there's no wildcard, the pattern and route should match exactly return route == pattern + @staticmethod + def _route_matches_allowed_route(route: str, allowed_route: str) -> bool: + """ + Check if route matches the allowed_route pattern. + Supports both exact match and prefix match. + + Examples: + - allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-6" -> True (exact match) + - allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-6/v1/chat/completions" -> True (prefix match) + - allowed_route="/fake-openai-proxy-6", route="/fake-openai-proxy-600" -> False (not a valid prefix) + + Args: + route: The actual route being accessed + allowed_route: The allowed route pattern + + Returns: + bool: True if route matches (exact or prefix), False otherwise + """ + # Exact match + if route == allowed_route: + return True + # Prefix match - ensure we add "/" to prevent false matches like /fake-openai-proxy-600 + if route.startswith(allowed_route + "/"): + return True + return False + @staticmethod def check_route_access(route: str, allowed_routes: List[str]) -> bool: """ @@ -429,6 +453,33 @@ class RouteChecks: return False + @staticmethod + def check_passthrough_route_access( + route: str, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + """ + Check if route is a passthrough route. + Supports both exact match and prefix match. + """ + metadata = user_api_key_dict.metadata + if metadata is None: + return False + if "allowed_passthrough_routes" not in metadata: + return False + if metadata["allowed_passthrough_routes"] is None: + return False + + allowed_passthrough_routes = metadata["allowed_passthrough_routes"] + + # Check if route matches any allowed passthrough route (exact or prefix match) + for allowed_route in allowed_passthrough_routes: + if RouteChecks._route_matches_allowed_route( + route=route, allowed_route=allowed_route + ): + return True + + return False + @staticmethod def _is_assistants_api_request(request: Request) -> bool: """ From 42f9d44753922271bf9c0e5c5d2509f0e4747b98 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 15:48:10 -0700 Subject: [PATCH 12/39] refactor: reduce function size below 50 LOC --- litellm/proxy/auth/route_checks.py | 52 +++++++++++-------- .../key_management_endpoints.py | 1 + 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py index 19d152cbdc7..96111434522 100644 --- a/litellm/proxy/auth/route_checks.py +++ b/litellm/proxy/auth/route_checks.py @@ -117,6 +117,32 @@ class RouteChecks: return masker._mask_value(user_id) + @staticmethod + def _raise_admin_only_route_exception( + user_obj: Optional[LiteLLM_UserTable], + route: str, + ) -> None: + """ + Raise exception for routes that require proxy admin access + + Args: + user_obj (Optional[LiteLLM_UserTable]): The user object + route (str): The route being accessed + + Raises: + Exception: With user role and masked user_id information + """ + user_role = "unknown" + user_id = "unknown" + if user_obj is not None: + user_role = user_obj.user_role or "unknown" + user_id = user_obj.user_id or "unknown" + + masked_user_id = RouteChecks._mask_user_id(user_id) + raise Exception( + f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" + ) + @staticmethod def non_proxy_admin_allowed_routes_check( user_obj: Optional[LiteLLM_UserTable], @@ -215,29 +241,13 @@ class RouteChecks: route_allowed = True break - if route_allowed: - pass - else: - user_role = "unknown" - user_id = "unknown" - if user_obj is not None: - user_role = user_obj.user_role or "unknown" - user_id = user_obj.user_id or "unknown" - - masked_user_id = RouteChecks._mask_user_id(user_id) - raise Exception( - f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" + if not route_allowed: + RouteChecks._raise_admin_only_route_exception( + user_obj=user_obj, route=route ) else: - user_role = "unknown" - user_id = "unknown" - if user_obj is not None: - user_role = user_obj.user_role or "unknown" - user_id = user_obj.user_id or "unknown" - - masked_user_id = RouteChecks._mask_user_id(user_id) - raise Exception( - f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={masked_user_id}" + RouteChecks._raise_admin_only_route_exception( + user_obj=user_obj, route=route ) @staticmethod diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 772cb42d3e6..aa486433396 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -1277,6 +1277,7 @@ async def update_key_fn( - temp_budget_increase: Optional[float] - Temporary budget increase for the key (Enterprise only). - temp_budget_expiry: Optional[str] - Expiry time for the temporary budget increase (Enterprise only). - allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"] + - allowed_passthrough_routes: Optional[list] - List of allowed pass through routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/my-custom-endpoint"]. Use this instead of allowed_routes, if you just want to specify which pass through routes the key can access, without specifying the routes. If allowed_routes is specified, allowed_passthrough_routes is ignored. - prompts: Optional[List[str]] - List of allowed prompts for the key. If specified, the key will only be able to use these specific prompts. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"], "mcp_tool_permissions": {"server_id_1": ["tool1", "tool2"]}}. IF null or {} then no object permission. - auto_rotate: Optional[bool] - Whether this key should be automatically rotated From 443b702f973dfca8c0ff7b5de1abd83609524770 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 16:14:59 -0700 Subject: [PATCH 13/39] feat(_types.py): support setting allowed passthrough routes on keys + teams ensures specific keys can call specific routes --- litellm/proxy/_types.py | 2 ++ .../management_endpoints/team_endpoints.py | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index babe3122fee..e338afc83a3 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1281,6 +1281,7 @@ class NewTeamRequest(TeamBase): guardrails: Optional[List[str]] = None prompts: Optional[List[str]] = None object_permission: Optional[LiteLLM_ObjectPermissionBase] = None + allowed_passthrough_routes: Optional[list] = None team_member_budget: Optional[float] = ( None # allow user to set a budget for all team members ) @@ -1336,6 +1337,7 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase): team_member_rpm_limit: Optional[int] = None team_member_tpm_limit: Optional[int] = None team_member_key_duration: Optional[str] = None + allowed_passthrough_routes: Optional[list] = None class ResetTeamBudgetRequest(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index d05a0259f46..0d02007b8b1 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -27,6 +27,7 @@ from litellm.proxy._types import ( CommonProxyErrors, DeleteTeamRequest, LiteLLM_AuditLogs, + LiteLLM_ManagementEndpoint_MetadataFields, LiteLLM_ManagementEndpoint_MetadataFields_Premium, LiteLLM_ModelTable, LiteLLM_OrganizationTable, @@ -56,9 +57,6 @@ from litellm.proxy._types import ( UpdateTeamRequest, UserAPIKeyAuth, ) -from litellm.proxy.management_helpers.object_permission_utils import ( - _set_object_permission, -) from litellm.proxy.auth.auth_checks import ( allowed_route_check_inside_route, can_org_access_model, @@ -76,6 +74,7 @@ from litellm.proxy.management_endpoints.tag_management_endpoints import ( get_daily_activity, ) from litellm.proxy.management_helpers.object_permission_utils import ( + _set_object_permission, handle_update_object_permission_common, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( @@ -321,6 +320,7 @@ async def new_team( # noqa: PLR0915 - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" - prompts: Optional[List[str]] - List of allowed prompts for the team. If specified, the team will only be able to use these specific prompts. + Returns: - team_id: (str) Unique team id - used for tracking spend across multiple keys for same team id. @@ -478,7 +478,7 @@ async def new_team( # noqa: PLR0915 ## Create Team Member Budget Table data_json = data.json() - + ## Handle Object Permission - MCP, Vector Stores etc. data_json = await _set_object_permission( data_json=data_json, @@ -514,6 +514,14 @@ async def new_team( # noqa: PLR0915 value=getattr(data, field), ) + for field in LiteLLM_ManagementEndpoint_MetadataFields: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=complete_team_data, + field_name=field, + value=getattr(data, field), + ) + # If budget_duration is set, set `budget_reset_at` if complete_team_data.budget_duration is not None: from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time @@ -619,7 +627,6 @@ async def _update_model_table( return _model_id - def validate_team_org_change( team: LiteLLM_TeamTable, organization: LiteLLM_OrganizationTable, llm_router: Router ) -> bool: @@ -877,6 +884,13 @@ async def update_team( field_name=field, ) + for field in LiteLLM_ManagementEndpoint_MetadataFields: + if field in updated_kv and updated_kv[field] is not None: + _update_team_metadata_field( + updated_kv=updated_kv, + field_name=field, + ) + if "model_aliases" in updated_kv: updated_kv.pop("model_aliases") _model_id = await _update_model_table( From c553bfbf663c169e345d8f591e51b940bab1f0b6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 16:29:30 -0700 Subject: [PATCH 14/39] feat(create_key_button.tsx): support setting passthrough endpoints by key on the UI --- .../organisms/create_key_button.tsx | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index b04d2e93aa1..1c7a07f1fd5 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -18,6 +18,7 @@ import { keyCreateServiceAccountCall, fetchMCPAccessGroups, getPromptsList, + getPassThroughEndpointsCall, } from "../networking"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; import { Team } from "../key_team_helpers/key_list"; @@ -159,6 +160,7 @@ const CreateKey: React.FC = ({ const [predefinedTags, setPredefinedTags] = useState(getPredefinedTags(data)); const [guardrailsList, setGuardrailsList] = useState([]); const [promptsList, setPromptsList] = useState([]); + const [passThroughRoutesList, setPassThroughRoutesList] = useState([]); const [loggingSettings, setLoggingSettings] = useState([]); const [selectedCreateKeyTeam, setSelectedCreateKeyTeam] = useState(team); const [isCreateUserModalVisible, setIsCreateUserModalVisible] = useState(false); @@ -240,8 +242,19 @@ const CreateKey: React.FC = ({ } }; + const fetchPassThroughEndpoints = async () => { + try { + const response = await getPassThroughEndpointsCall(accessToken); + const passThroughPaths = response.endpoints.map((endpoint: { path: string }) => endpoint.path); + setPassThroughRoutesList(passThroughPaths); + } catch (error) { + console.error("Failed to fetch pass through endpoints:", error); + } + }; + fetchGuardrails(); fetchPrompts(); + fetchPassThroughEndpoints(); }, [accessToken]); // Fetch possible user roles when component mounts @@ -914,6 +927,40 @@ const CreateKey: React.FC = ({ options={promptsList.map((name) => ({ value: name, label: name }))} /> + + Allowed Pass Through Routes{" "} + + e.stopPropagation()} // Prevent accordion from collapsing when clicking link + > + + + + + } + name="allowed_passthrough_routes" + className="mt-4" + help={ + premiumUser + ? "Select existing pass through routes or enter new ones" + : "Premium feature - Upgrade to set pass through routes by key" + } + > + 0 + ? `Current: ${keyData.metadata.allowed_passthrough_routes.join(", ")}` + : "Select or enter allowed pass through routes" + } + options={passThroughRoutesList.map((name) => ({ value: name, label: name }))} + /> + + + form.setFieldValue("vector_stores", values)} diff --git a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx index e6ae7e6639e..1d1b472dd1f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_info_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_info_view.tsx @@ -642,6 +642,19 @@ export default function KeyInfoView({ +
+ Allowed Pass Through Routes + + {Array.isArray(currentKeyData.metadata?.allowed_passthrough_routes) && currentKeyData.metadata.allowed_passthrough_routes.length > 0 + ? currentKeyData.metadata.allowed_passthrough_routes.map((route, index) => ( + + {route} + + )) + : "No pass through routes specified"} + +
+
Models
From f608aefc2d1d2ec35cc77ffbd64fedf41169a9d4 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 16:36:39 -0700 Subject: [PATCH 16/39] feat(proxy/_types.py): make setting allowed passthrough routes a premium field on key/team --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index e338afc83a3..8895aaabd0f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3108,7 +3108,6 @@ LiteLLM_ManagementEndpoint_MetadataFields = [ "enforced_params", "temp_budget_increase", "temp_budget_expiry", - "allowed_passthrough_routes", ] LiteLLM_ManagementEndpoint_MetadataFields_Premium = [ @@ -3117,6 +3116,7 @@ LiteLLM_ManagementEndpoint_MetadataFields_Premium = [ "team_member_key_duration", "prompts", "logging", + "allowed_passthrough_routes", ] From 82d7a7248e91f3fdf9b46a4842659e4d9db99f24 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 Oct 2025 17:11:57 -0700 Subject: [PATCH 17/39] feat(teams/): support allowed_passthrough_routes on team create + update allows admin to specify what passthrough routes the team has access to --- .../PassThroughRoutesSelector.tsx | 67 +++++++++++++++++++ .../organisms/create_key_button.tsx | 28 +++----- .../src/components/team/team_info.tsx | 10 +++ ui/litellm-dashboard/src/components/teams.tsx | 21 ++++++ .../components/templates/key_edit_view.tsx | 25 ++----- 5 files changed, 113 insertions(+), 38 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx diff --git a/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx new file mode 100644 index 00000000000..a2ed9c6d66b --- /dev/null +++ b/ui/litellm-dashboard/src/components/common_components/PassThroughRoutesSelector.tsx @@ -0,0 +1,67 @@ +import React, { useEffect, useState } from "react"; +import { Select } from "antd"; +import { getPassThroughEndpointsCall } from "../networking"; + +interface PassThroughRoutesSelectorProps { + onChange: (selectedRoutes: string[]) => void; + value?: string[]; + className?: string; + accessToken: string; + placeholder?: string; + disabled?: boolean; +} + +const PassThroughRoutesSelector: React.FC = ({ + onChange, + value, + className, + accessToken, + placeholder = "Select pass through routes", + disabled = false, +}) => { + const [passThroughRoutes, setPassThroughRoutes] = useState([]); + const [loading, setLoading] = useState(false); + + useEffect(() => { + const fetchPassThroughRoutes = async () => { + if (!accessToken) return; + + setLoading(true); + try { + const response = await getPassThroughEndpointsCall(accessToken); + if (response.endpoints) { + const routes = response.endpoints.map((route: { path: string }) => route.path); + setPassThroughRoutes(routes); + } + } catch (error) { + console.error("Error fetching pass through routes:", error); + } finally { + setLoading(false); + } + }; + + fetchPassThroughRoutes(); + }, [accessToken]); + + return ( + form.setFieldValue("allowed_passthrough_routes", values)} + value={form.getFieldValue("allowed_passthrough_routes")} + accessToken={accessToken} placeholder={ !premiumUser ? "Premium feature - Upgrade to set pass through routes by key" : "Select or enter pass through routes" } - options={passThroughRoutesList.map((name) => ({ value: name, label: name }))} + disabled={!premiumUser} /> = ({ /> + + form.setFieldValue("allowed_passthrough_routes", values)} + value={form.getFieldValue("allowed_passthrough_routes")} + accessToken={accessToken || ""} + placeholder="Select pass through routes" + /> + + form.setFieldValue("mcp_servers_and_groups", val)} diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index 590e0952944..a7399ba4230 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -63,6 +63,7 @@ import AvailableTeamsPanel from "@/components/team/available_teams"; import VectorStoreSelector from "./vector_store_management/VectorStoreSelector"; import PremiumLoggingSettings from "./common_components/PremiumLoggingSettings"; import type { KeyResponse, Team } from "./key_team_helpers/key_list"; +import PassThroughRoutesSelector from "./common_components/PassThroughRoutesSelector"; import { formatNumberWithCommas } from "../utils/dataUtils"; import { AlertTriangleIcon, XIcon } from "lucide-react"; import MCPServerSelector from "./mcp_server_management/MCPServerSelector"; @@ -1251,6 +1252,26 @@ const Teams: React.FC = ({ placeholder="Select vector stores (optional)" /> + + Allowed Pass Through Routes{" "} + + + + + } + name="allowed_passthrough_routes" + className="mt-8" + help="Select pass through routes this team can access. Leave empty for access to all pass through routes" + > + form.setFieldValue("allowed_passthrough_routes", values)} + value={form.getFieldValue("allowed_passthrough_routes")} + accessToken={accessToken || ""} + placeholder="Select pass through routes (optional)" + /> + diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx index 04dbee673b4..05c6acfe68f 100644 --- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx +++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx @@ -3,7 +3,7 @@ import { Form, Input, Select, Button as AntdButton, Tooltip } from "antd"; import { Button as TremorButton, TextInput } from "@tremor/react"; import { KeyResponse } from "../key_team_helpers/key_list"; import { fetchTeamModels } from "../organisms/create_key_button"; -import { modelAvailableCall, getPromptsList, getPassThroughEndpointsCall } from "../networking"; +import { modelAvailableCall, getPromptsList } from "../networking"; import NumericalInput from "../shared/numerical_input"; import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; @@ -15,6 +15,7 @@ import { mapInternalToDisplayNames, mapDisplayToInternalNames } from "../callbac import GuardrailSelector from "@/components/guardrails/GuardrailSelector"; import KeyLifecycleSettings from "../common_components/KeyLifecycleSettings"; import RateLimitTypeFormItem from "../common_components/RateLimitTypeFormItem"; +import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; interface KeyEditViewProps { keyData: KeyResponse; @@ -59,7 +60,6 @@ export function KeyEditView({ const [form] = Form.useForm(); const [userModels, setUserModels] = useState([]); const [promptsList, setPromptsList] = useState([]); - const [passThroughRoutesList, setPassThroughRoutesList] = useState([]); const team = teams?.find((team) => team.team_id === keyData.team_id); const [availableModels, setAvailableModels] = useState([]); const [mcpAccessGroups, setMcpAccessGroups] = useState([]); @@ -114,19 +114,8 @@ export function KeyEditView({ } }; - const fetchPassThroughRoutes = async () => { - if (!accessToken) return; - try { - const response = await getPassThroughEndpointsCall(accessToken); - setPassThroughRoutesList(response.endpoints.map((route: { path: string }) => route.path)); - } catch (error) { - console.error("Failed to fetch pass through routes:", error); - } - }; - fetchPrompts(); fetchModels(); - fetchPassThroughRoutes(); }, [userID, userRole, accessToken, team, keyData.team_id]); // Sync disabled callbacks with form when component mounts @@ -290,10 +279,10 @@ export function KeyEditView({ - { } }; -export const getPassThroughEndpointsCall = async (accessToken: String) => { +export const getPassThroughEndpointsCall = async (accessToken: String, teamId?: string | null) => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint` : `/config/pass_through_endpoint`; + if (teamId) { + url += `/team/${teamId}`; + } + //NotificationsManager.info("Requesting model data"); const response = await fetch(url, { method: "GET", diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx index 60b12c14310..896b7301c6c 100644 --- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx @@ -947,6 +947,7 @@ const CreateKey: React.FC = ({ !premiumUser ? "Premium feature - Upgrade to set pass through routes by key" : "Select or enter pass through routes" } disabled={!premiumUser} + teamId={selectedCreateKeyTeam ? selectedCreateKeyTeam.team_id : null} /> Date: Fri, 10 Oct 2025 14:53:15 -0700 Subject: [PATCH 20/39] feat(responses_api/): fix missing streaming events on responses api <-> chat completion bridge ensure we are passing the required events when streaming non-openai models via responses api --- ...odel_prices_and_context_window_backup.json | 33 ++++ .../index.html} | 0 .../index.html} | 0 .../out/{logs.html => logs/index.html} | 0 .../{model-hub.html => model-hub/index.html} | 0 .../index.html} | 0 .../index.html} | 0 .../proxy/_experimental/out/onboarding.html | 1 - .../index.html} | 0 .../out/{teams.html => teams/index.html} | 0 .../{test-key.html => test-key/index.html} | 0 .../out/{usage.html => usage/index.html} | 0 .../out/{users.html => users/index.html} | 0 .../index.html} | 0 litellm/proxy/_new_secret_config.yaml | 39 ++-- .../handler.py | 10 +- .../streaming_iterator.py | 167 +++++++++++++++++- litellm/types/llms/openai.py | 16 +- .../test_anthropic_completion.py | 124 +++++++++++++ .../test_reasoning_content_transformation.py | 13 +- 20 files changed, 361 insertions(+), 42 deletions(-) rename litellm/proxy/_experimental/out/{api-reference.html => api-reference/index.html} (100%) rename litellm/proxy/_experimental/out/{guardrails.html => guardrails/index.html} (100%) rename litellm/proxy/_experimental/out/{logs.html => logs/index.html} (100%) rename litellm/proxy/_experimental/out/{model-hub.html => model-hub/index.html} (100%) rename litellm/proxy/_experimental/out/{model_hub_table.html => model_hub_table/index.html} (100%) rename litellm/proxy/_experimental/out/{models-and-endpoints.html => models-and-endpoints/index.html} (100%) delete mode 100644 litellm/proxy/_experimental/out/onboarding.html rename litellm/proxy/_experimental/out/{organizations.html => organizations/index.html} (100%) rename litellm/proxy/_experimental/out/{teams.html => teams/index.html} (100%) rename litellm/proxy/_experimental/out/{test-key.html => test-key/index.html} (100%) rename litellm/proxy/_experimental/out/{usage.html => usage/index.html} (100%) rename litellm/proxy/_experimental/out/{users.html => users/index.html} (100%) rename litellm/proxy/_experimental/out/{virtual-keys.html => virtual-keys/index.html} (100%) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6d49d6e7e2d..38cf9671686 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -19904,6 +19904,39 @@ "supports_parallel_function_calling": true, "supports_tool_choice": true }, + "together_ai/moonshotai/Kimi-K2-Instruct-0905": { + "input_cost_per_token": 1e-06, + "litellm_provider": "together_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://www.together.ai/models/kimi-k2-0905", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true + }, + "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true + }, + "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_tool_choice": true + }, "tts-1": { "input_cost_per_character": 1.5e-05, "litellm_provider": "openai", diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails/index.html similarity index 100% rename from litellm/proxy/_experimental/out/guardrails.html rename to litellm/proxy/_experimental/out/guardrails/index.html diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 5df786e1f14..00000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 95c6869b95b..5feb7797974 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,29 +1,22 @@ model_list: - - model_name: gpt-5-mini + - model_name: openai/gpt-4o-mini litellm_params: model: openai/gpt-4o-mini - api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" - api_key: dummy - - model_name: "byok-wildcard/*" + api_key: os.environ/OPENAI_API_KEY + - model_name: vertex/gemini-2.5-flash litellm_params: - model: openai/* - - model_name: xai-grok-3 + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY + - model_name: anthropic/claude-sonnet-4-5 litellm_params: - model: xai/grok-3 - - model_name: hosted_vllm/whisper-v3 + model: anthropic/claude-sonnet-4-5 + api_key: os.environ/ANTHROPIC_API_KEY + +guardrails: + - guardrail_name: "bedrock-guardrail" litellm_params: - model: hosted_vllm/whisper-v3 - api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" - api_key: dummy - -mcp_servers: - my_api_mcp: - url: "http://0.0.0.0:8090" - spec_path: "/Users/krrishdholakia/Documents/temp_py_folder/example_openapi.json" - auth_type: none - allowed_tools: ["getpetbyid", "my_api_mcp-findpetsbystatus"] - - -litellm_settings: - callbacks: ["prometheus"] - custom_prometheus_metadata_labels: ["metadata.initiative", "metadata.business-unit"] + guardrail: bedrock + mode: "post_call" + guardrailIdentifier: gf3sc1mzinjw + guardrailVersion: "DRAFT" + disable_exception_on_block: true # Prevents exceptions when content is blocked \ No newline at end of file diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 9317bf26178..7e0b4cfa243 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -58,7 +58,7 @@ class LiteLLMCompletionTransformationHandler: responses_api_request=responses_api_request, **kwargs, ) - + completion_args = {} completion_args.update(kwargs) completion_args.update(litellm_completion_request) @@ -83,6 +83,7 @@ class LiteLLMCompletionTransformationHandler: elif isinstance(litellm_completion_response, litellm.CustomStreamWrapper): return LiteLLMCompletionStreamingIterator( + model=model, litellm_custom_stream_wrapper=litellm_completion_response, request_input=input, responses_api_request=responses_api_request, @@ -106,7 +107,7 @@ class LiteLLMCompletionTransformationHandler: previous_response_id=previous_response_id, litellm_completion_request=litellm_completion_request, ) - + acompletion_args = {} acompletion_args.update(kwargs) acompletion_args.update(litellm_completion_request) @@ -130,9 +131,12 @@ class LiteLLMCompletionTransformationHandler: elif isinstance(litellm_completion_response, litellm.CustomStreamWrapper): return LiteLLMCompletionStreamingIterator( + model=litellm_completion_request.get("model") or "", litellm_custom_stream_wrapper=litellm_completion_response, request_input=request_input, responses_api_request=responses_api_request, - custom_llm_provider=litellm_completion_request.get("custom_llm_provider"), + custom_llm_provider=litellm_completion_request.get( + "custom_llm_provider" + ), litellm_metadata=kwargs.get("litellm_metadata", {}), ) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 20822b3628e..af2d388304f 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,3 +1,5 @@ +import time +import uuid from typing import List, Optional, Union import litellm @@ -8,11 +10,17 @@ from litellm.responses.litellm_completion_transformation.transformation import ( from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( + BaseLiteLLMOpenAIResponseObject, + ContentPartAddedEvent, + OutputItemAddedEvent, OutputTextDeltaEvent, ReasoningSummaryTextDeltaEvent, ResponseCompletedEvent, + ResponseCreatedEvent, + ResponseInProgressEvent, ResponseInputParam, ResponsesAPIOptionalRequestParams, + ResponsesAPIResponse, ResponsesAPIStreamEvents, ResponsesAPIStreamingResponse, ) @@ -32,12 +40,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def __init__( self, + model: str, litellm_custom_stream_wrapper: litellm.CustomStreamWrapper, request_input: Union[str, ResponseInputParam], responses_api_request: ResponsesAPIOptionalRequestParams, custom_llm_provider: Optional[str] = None, litellm_metadata: Optional[dict] = None, ): + self.model: str = model self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = ( litellm_custom_stream_wrapper ) @@ -50,14 +60,139 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.collected_chat_completion_chunks: List[ModelResponseStream] = [] self.finished: bool = False self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj + self.sent_response_created_event: bool = False + self.sent_response_in_progress_event: bool = False + self.sent_output_item_added_event: bool = False + self.sent_content_part_added_event: bool = False + + def _default_response_created_event_data(self) -> dict: + response_created_event_data = { + "id": f"resp_{str(uuid.uuid4())}", + "object": "response", + "created_at": int(time.time()), + "status": "in_progress", + "error": None, + "incomplete_details": None, + "instructions": self.request_input, + "max_output_tokens": None, + "model": self.model, + "output": [], + "parallel_tool_calls": True, + "previous_response_id": None, + "reasoning": {"effort": None, "summary": None}, + "store": True, + } + if "temperature" in self.responses_api_request: + response_created_event_data["temperature"] = self.responses_api_request[ + "temperature" + ] + if "text" in self.responses_api_request: + response_created_event_data["text"] = self.responses_api_request["text"] + if "tool_choice" in self.responses_api_request: + response_created_event_data["tool_choice"] = self.responses_api_request[ + "tool_choice" + ] + else: + response_created_event_data["tool_choice"] = "auto" + if "tools" in self.responses_api_request: + response_created_event_data["tools"] = self.responses_api_request["tools"] + else: + response_created_event_data["tools"] = [] + if "top_p" in self.responses_api_request: + response_created_event_data["top_p"] = self.responses_api_request["top_p"] + else: + response_created_event_data["top_p"] = 1.0 + if "truncation" in self.responses_api_request: + response_created_event_data["truncation"] = self.responses_api_request[ + "truncation" + ] + if "usage" in self.responses_api_request: + response_created_event_data["usage"] = self.responses_api_request["usage"] + if "user" in self.responses_api_request: + response_created_event_data["user"] = self.responses_api_request["user"] + if "metadata" in self.responses_api_request: + response_created_event_data["metadata"] = self.responses_api_request[ + "metadata" + ] + return response_created_event_data + + def create_response_created_event(self) -> ResponseCreatedEvent: + """ + data: {"type":"response.created","response":{"id":"resp_67c9fdcecf488190bdd9a0409de3a1ec07b8b0ad4e5eb654","object":"response","created_at":1741290958,"status":"in_progress","error":null,"incomplete_details":null,"instructions":"You are a helpful assistant.","max_output_tokens":null,"model":"gpt-4.1-2025-04-14","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + + """ + response_created_event_data = self._default_response_created_event_data() + return ResponseCreatedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_CREATED, + response=ResponsesAPIResponse(**response_created_event_data), + ) + + def create_response_in_progress_event(self) -> ResponseInProgressEvent: + response_in_progress_event_data = self._default_response_created_event_data() + response_in_progress_event_data["status"] = "in_progress" + return ResponseInProgressEvent( + type=ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, + response=ResponsesAPIResponse(**response_in_progress_event_data), + ) + + def create_output_item_added_event(self) -> OutputItemAddedEvent: + return OutputItemAddedEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + output_index=0, + item=BaseLiteLLMOpenAIResponseObject( + **{ + "id": f"msg_{str(uuid.uuid4())}", + "type": "message", + "status": "in_progress", + "role": "assistant", + "content": [], + } + ), + ) + + def create_content_part_added_event(self) -> ContentPartAddedEvent: + return ContentPartAddedEvent( + type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED, + item_id=f"msg_{str(uuid.uuid4())}", + output_index=0, + content_index=0, + part=BaseLiteLLMOpenAIResponseObject( + **{"type": "output_text", "text": "", "annotations": []} + ), + ) + + def return_default_initial_events( + self, + ) -> Optional[BaseLiteLLMOpenAIResponseObject]: + if self.sent_response_created_event is False: + self.sent_response_created_event = True + return self.create_response_created_event() + elif self.sent_response_in_progress_event is False: + self.sent_response_in_progress_event = True + return self.create_response_in_progress_event() + elif self.sent_output_item_added_event is False: + self.sent_output_item_added_event = True + return self.create_output_item_added_event() + elif self.sent_content_part_added_event is False: + self.sent_content_part_added_event = True + return self.create_content_part_added_event() + return None async def __anext__( self, - ) -> Union[ResponsesAPIStreamingResponse, ResponseCompletedEvent]: + ) -> Union[ + ResponsesAPIStreamingResponse, + ResponseCompletedEvent, + BaseLiteLLMOpenAIResponseObject, + ]: try: while True: if self.finished is True: raise StopAsyncIteration + + result = self.return_default_initial_events() + if result: + return result # Get the next chunk from the stream try: chunk = await self.litellm_custom_stream_wrapper.__anext__() @@ -87,12 +222,20 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def __next__( self, - ) -> Union[ResponsesAPIStreamingResponse, ResponseCompletedEvent]: + ) -> Union[ + ResponsesAPIStreamingResponse, + ResponseCompletedEvent, + BaseLiteLLMOpenAIResponseObject, + ]: try: while True: if self.finished is True: raise StopIteration # Get the next chunk from the stream + + result = self.return_default_initial_events() + if result: + return result try: chunk = self.litellm_custom_stream_wrapper.__next__() self.collected_chat_completion_chunks.append(chunk) @@ -168,23 +311,33 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): def _emit_response_completed_event(self) -> Optional[ResponseCompletedEvent]: litellm_model_response: Optional[ Union[ModelResponse, TextCompletionResponse] - ] = stream_chunk_builder(chunks=self.collected_chat_completion_chunks, logging_obj=self.litellm_logging_obj) + ] = stream_chunk_builder( + chunks=self.collected_chat_completion_chunks, + logging_obj=self.litellm_logging_obj, + ) if litellm_model_response and isinstance(litellm_model_response, ModelResponse): # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None: + if ( + litellm.include_cost_in_streaming_usage + and self.litellm_logging_obj is not None + ): usage = getattr(litellm_model_response, "usage", None) if usage is not None: setattr( - usage, "cost", self.litellm_logging_obj._response_cost_calculator(result=litellm_model_response) + usage, + "cost", + self.litellm_logging_obj._response_cost_calculator( + result=litellm_model_response + ), ) - + # Transform the response responses_api_response = LiteLLMCompletionResponsesConfig.transform_chat_completion_response_to_responses_api_response( request_input=self.request_input, chat_completion_response=litellm_model_response, responses_api_request=self.responses_api_request, ) - + # Encode the response ID to match non-streaming behavior encoded_response = ResponsesAPIRequestUtils._update_responses_api_response_id_with_model_id( responses_api_response=responses_api_response, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0369ed1e204..1dae91f94c5 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,3 +1,4 @@ +import uuid from enum import Enum from os import PathLike from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union @@ -44,7 +45,7 @@ from openai.types.responses.response import ( # Handle OpenAI SDK version compatibility for Text type try: # fmt: off - from openai.types.responses.response_create_params import ( # type: ignore[attr-defined] + from openai.types.responses.response_create_params import ( Text as ResponseText, # type: ignore[attr-defined] ) @@ -992,7 +993,9 @@ class ResponsesAPIOptionalRequestParams(TypedDict, total=False): prompt_cache_key: Optional[str] stream_options: Optional[dict] top_logprobs: Optional[int] - partial_images: Optional[int] # Number of partial images to generate (1-3) for streaming image generation + partial_images: Optional[ + int + ] # Number of partial images to generate (1-3) for streaming image generation class ResponsesAPIRequestParams(ResponsesAPIOptionalRequestParams, total=False): @@ -1180,13 +1183,13 @@ class ReasoningSummaryTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED] output_index: int - item: Optional[dict] + item: Optional[BaseLiteLLMOpenAIResponseObject] class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] output_index: int - item: dict + item: BaseLiteLLMOpenAIResponseObject class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): @@ -1194,7 +1197,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): item_id: str output_index: int content_index: int - part: dict + part: BaseLiteLLMOpenAIResponseObject class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): @@ -1202,7 +1205,7 @@ class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): item_id: str output_index: int content_index: int - part: dict + part: BaseLiteLLMOpenAIResponseObject class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): @@ -1413,6 +1416,7 @@ ResponsesAPIStreamingResponse = Annotated[ ImageGenerationPartialImageEvent, ErrorEvent, GenericEvent, + BaseLiteLLMOpenAIResponseObject, ], Discriminator("type"), ] diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 4eeb80c4191..7d9ef56f164 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -361,6 +361,7 @@ def test_process_anthropic_headers_with_no_matching_headers(): def test_anthropic_tool_use(tool_type, tool_config, message_content): """Test Anthropic tool use with computer use and web fetch tools.""" from litellm import completion + litellm._turn_on_debug() tools = [tool_config] @@ -1518,3 +1519,126 @@ def test_anthropic_streaming(): role_set_count += 1 assert role_set_count == 1 + + +def test_anthropic_via_responses_api(): + from litellm.types.llms.openai import ResponsesAPIStreamEvents + + response = litellm.responses( + model="anthropic/claude-sonnet-4-5", + input="Who won the World Cup in 2022?", + max_output_tokens=100, + stream=True, + ) + + assert response is not None + + # Expected event sequence + expected_events = [ + ResponsesAPIStreamEvents.RESPONSE_CREATED, + ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, + ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, + ResponsesAPIStreamEvents.CONTENT_PART_ADDED, + ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, # Can occur multiple times + ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, + ResponsesAPIStreamEvents.CONTENT_PART_DONE, + ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + ] + + events_seen = [] + text_delta_count = 0 + + for chunk in response: + print(f"chunk: {chunk}") + + # Each chunk should have a type attribute + assert hasattr(chunk, "type"), f"Chunk missing 'type' attribute: {chunk}" + + event_type = chunk.type + + # Track events seen + if event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA: + text_delta_count += 1 + if ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA not in events_seen: + events_seen.append(event_type) + else: + events_seen.append(event_type) + + # Assert specific structures for each event type + if event_type == ResponsesAPIStreamEvents.RESPONSE_CREATED: + assert chunk.type == ResponsesAPIStreamEvents.RESPONSE_CREATED + assert hasattr(chunk, "response") + assert chunk.response.status == "in_progress" + assert hasattr(chunk.response, "id") + assert hasattr(chunk.response, "model") + + elif event_type == ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS: + assert chunk.type == ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS + assert hasattr(chunk, "response") + assert chunk.response.status == "in_progress" + + elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED: + assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + assert hasattr(chunk, "output_index") + assert hasattr(chunk, "item") + assert chunk.item.type == "message" + assert chunk.item.role == "assistant" + + elif event_type == ResponsesAPIStreamEvents.CONTENT_PART_ADDED: + assert chunk.type == ResponsesAPIStreamEvents.CONTENT_PART_ADDED + assert hasattr(chunk, "item_id") + assert hasattr(chunk, "output_index") + assert hasattr(chunk, "content_index") + assert hasattr(chunk, "part") + assert chunk.part.type == "output_text" + + elif event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA: + assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA + assert hasattr(chunk, "item_id") + assert hasattr(chunk, "output_index") + assert hasattr(chunk, "content_index") + assert hasattr(chunk, "delta") + assert isinstance(chunk.delta, str) + + elif event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE: + assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE + assert hasattr(chunk, "item_id") + assert hasattr(chunk, "output_index") + assert hasattr(chunk, "content_index") + assert hasattr(chunk, "text") + + elif event_type == ResponsesAPIStreamEvents.CONTENT_PART_DONE: + assert chunk.type == ResponsesAPIStreamEvents.CONTENT_PART_DONE + assert hasattr(chunk, "item_id") + assert hasattr(chunk, "output_index") + assert hasattr(chunk, "content_index") + assert hasattr(chunk, "part") + assert chunk.part.type == "output_text" + + elif event_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + assert chunk.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE + assert hasattr(chunk, "output_index") + assert hasattr(chunk, "item") + assert chunk.item.status == "completed" + + elif event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED: + assert chunk.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + assert hasattr(chunk, "response") + assert chunk.response.status == "completed" + assert hasattr(chunk.response, "usage") + assert hasattr(chunk.response, "output") + + # Assert we saw all expected events + print(f"Events seen: {events_seen}") + assert ( + events_seen == expected_events + ), f"Event sequence mismatch. Expected: {expected_events}, Got: {events_seen}" + + # Assert we saw at least one text delta + assert ( + text_delta_count > 0 + ), f"Expected at least one response.output_text.delta event, got {text_delta_count}" + + print(f"✓ All {len(events_seen)} events matched expected structure") + print(f"✓ Received {text_delta_count} text delta chunks") diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py index 5323589818b..d1926bbc93f 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py @@ -4,14 +4,20 @@ Test reasoning content preservation in Responses API transformation from unittest.mock import AsyncMock -from litellm.types.utils import ModelResponseStream, StreamingChoices, Delta from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) -from litellm.types.utils import ModelResponse, Choices, Message +from litellm.types.utils import ( + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) class TestReasoningContentStreaming: @@ -41,6 +47,7 @@ class TestReasoningContentStreaming: mock_stream = AsyncMock() iterator = LiteLLMCompletionStreamingIterator( + model="test-model", litellm_custom_stream_wrapper=mock_stream, request_input="Test input", responses_api_request={}, @@ -78,6 +85,7 @@ class TestReasoningContentStreaming: mock_stream = AsyncMock() iterator = LiteLLMCompletionStreamingIterator( + model="test-model", litellm_custom_stream_wrapper=mock_stream, request_input="Test input", responses_api_request={}, @@ -114,6 +122,7 @@ class TestReasoningContentStreaming: mock_stream = AsyncMock() iterator = LiteLLMCompletionStreamingIterator( + model="test-model", litellm_custom_stream_wrapper=mock_stream, request_input="Test input", responses_api_request={}, From c74eb6403d540f67456a4800a72b2389969498e0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 Oct 2025 15:30:10 -0700 Subject: [PATCH 21/39] fix(streaming_iterator.py): return done chunks in the order expected by responses api ensures responses api sdk's (e.g. openai ruby) work when calling non-openai models --- .../streaming_iterator.py | 171 +++++++++++++++--- litellm/types/llms/openai.py | 50 +++-- 2 files changed, 185 insertions(+), 36 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index af2d388304f..e0cbea501f9 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,6 +1,6 @@ import time import uuid -from typing import List, Optional, Union +from typing import List, Optional, Union, cast import litellm from litellm.main import stream_chunk_builder @@ -12,8 +12,13 @@ from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, ContentPartAddedEvent, + ContentPartDoneEvent, + ContentPartDonePartOutputText, + ContentPartDonePartReasoningText, OutputItemAddedEvent, + OutputItemDoneEvent, OutputTextDeltaEvent, + OutputTextDoneEvent, ReasoningSummaryTextDeltaEvent, ResponseCompletedEvent, ResponseCreatedEvent, @@ -64,6 +69,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.sent_response_in_progress_event: bool = False self.sent_output_item_added_event: bool = False self.sent_content_part_added_event: bool = False + self.sent_output_text_done_event: bool = False + self.sent_output_content_part_done_event: bool = False + self.sent_output_item_done_event: bool = False + self.litellm_model_response: Optional[ + Union[ModelResponse, TextCompletionResponse] + ] = None + self.final_text: str = "" def _default_response_created_event_data(self) -> dict: response_created_event_data = { @@ -161,6 +173,97 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ), ) + def create_litellm_model_response( + self, + ) -> Optional[ModelResponse]: + return cast( + Optional[ModelResponse], + stream_chunk_builder( + chunks=self.collected_chat_completion_chunks, + logging_obj=self.litellm_logging_obj, + ), + ) + + def create_output_text_done_event( + self, litellm_complete_object: ModelResponse + ) -> OutputTextDoneEvent: + return OutputTextDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, + item_id=f"msg_{str(uuid.uuid4())}", + output_index=0, + content_index=0, + text=getattr(litellm_complete_object.choices[0].message, "content", "") # type: ignore + or "", + ) + + def create_output_content_part_done_event( + self, litellm_complete_object: ModelResponse + ) -> ContentPartDoneEvent: + + text = getattr(litellm_complete_object.choices[0].message, "content", "") or "" # type: ignore + reasoning_content = getattr(litellm_complete_object.choices[0].message, "reasoning_content", "") or "" # type: ignore + + if reasoning_content: + part = ContentPartDonePartReasoningText( + type="reasoning_text", + reasoning=reasoning_content, + ) + + else: + part = ContentPartDonePartOutputText( + type="output_text", + text=text, + annotations=[], + logprobs=None, + ) + + return ContentPartDoneEvent( + type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, + item_id=f"msg_{str(uuid.uuid4())}", + output_index=0, + content_index=0, + part=part, + ) + + def create_output_item_done_event( + self, litellm_complete_object: ModelResponse + ) -> OutputItemDoneEvent: + text = self.litellm_model_response.choices[0].message.content or "" # type: ignore + return OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=0, + sequence_number=1, + item=BaseLiteLLMOpenAIResponseObject( + **{ + "id": f"msg_{str(uuid.uuid4())}", + "status": "completed", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": text, + "annotations": [], + } + ], + } + ), + ) + + def return_default_done_events( + self, litellm_complete_object: ModelResponse + ) -> Optional[BaseLiteLLMOpenAIResponseObject]: + if self.sent_output_text_done_event is False: + self.sent_output_text_done_event = True + return self.create_output_text_done_event(litellm_complete_object) + if self.sent_output_content_part_done_event is False: + self.sent_output_content_part_done_event = True + return self.create_output_content_part_done_event(litellm_complete_object) + if self.sent_output_item_done_event is False: + self.sent_output_item_done_event = True + return self.create_output_item_done_event(litellm_complete_object) + return None + def return_default_initial_events( self, ) -> Optional[BaseLiteLLMOpenAIResponseObject]: @@ -178,6 +281,44 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return self.create_content_part_added_event() return None + def is_stream_finished(self) -> bool: + if ( + self.sent_output_text_done_event is True + and self.sent_output_content_part_done_event is True + and self.sent_output_item_done_event is True + ): + return True + return False + + def common_done_event_logic( + self, sync_mode: bool = True + ) -> BaseLiteLLMOpenAIResponseObject: + if not self.litellm_model_response or isinstance( + self.litellm_model_response, TextCompletionResponse + ): + self.litellm_model_response = self.create_litellm_model_response() + if self.litellm_model_response: + done_event = self.return_default_done_events(self.litellm_model_response) + if done_event: + return done_event + else: + if sync_mode: + raise StopIteration + else: + raise StopAsyncIteration + + self.finished = self.is_stream_finished() + response_completed_event = self._emit_response_completed_event( + self.litellm_model_response + ) + if response_completed_event: + return response_completed_event + else: + if sync_mode: + raise StopIteration + else: + raise StopAsyncIteration + async def __anext__( self, ) -> Union[ @@ -205,12 +346,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if response_api_chunk: return response_api_chunk except StopAsyncIteration: - self.finished = True - response_completed_event = self._emit_response_completed_event() - if response_completed_event: - return response_completed_event - else: - raise StopAsyncIteration + return self.common_done_event_logic(sync_mode=False) except Exception as e: # Handle HTTP errors @@ -247,13 +383,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if response_api_chunk: return response_api_chunk except StopIteration: - self.finished = True - response_completed_event = self._emit_response_completed_event() - if response_completed_event: - return response_completed_event - else: - raise StopIteration - + return self.common_done_event_logic(sync_mode=True) except Exception as e: # Handle HTTP errors self.finished = True @@ -308,14 +438,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): chat_completion_delta: ChatCompletionDelta = choice.delta return chat_completion_delta.content or "" - def _emit_response_completed_event(self) -> Optional[ResponseCompletedEvent]: - litellm_model_response: Optional[ - Union[ModelResponse, TextCompletionResponse] - ] = stream_chunk_builder( - chunks=self.collected_chat_completion_chunks, - logging_obj=self.litellm_logging_obj, - ) - if litellm_model_response and isinstance(litellm_model_response, ModelResponse): + def _emit_response_completed_event( + self, litellm_model_response: ModelResponse + ) -> Optional[ResponseCompletedEvent]: + + if litellm_model_response: # Add cost to usage object if include_cost_in_streaming_usage is True if ( litellm.include_cost_in_streaming_usage diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 1dae91f94c5..b1f43efedf3 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1189,9 +1189,23 @@ class OutputItemAddedEvent(BaseLiteLLMOpenAIResponseObject): class OutputItemDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE] output_index: int + sequence_number: int = 1 item: BaseLiteLLMOpenAIResponseObject +class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False): + bytes: List + logprob: Required[float] + token: Required[str] + + +class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): + bytes: List + logprob: Required[float] + token: Required[str] + top_logprobs: List[OpenAIChatCompletionLogprobsContentTopLogprobs] + + class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_ADDED] item_id: str @@ -1200,12 +1214,33 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject): part: BaseLiteLLMOpenAIResponseObject +class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject): + type: Literal["output_text"] + text: str + annotations: List[BaseLiteLLMOpenAIResponseObject] + logprobs: Optional[List[OpenAIChatCompletionLogprobsContent]] + + +class ContentPartDonePartRefusal(BaseLiteLLMOpenAIResponseObject): + type: Literal["refusal"] + refusal: str + + +class ContentPartDonePartReasoningText(BaseLiteLLMOpenAIResponseObject): + type: Literal["reasoning_text"] + reasoning: str + + class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] item_id: str output_index: int content_index: int - part: BaseLiteLLMOpenAIResponseObject + part: Union[ + ContentPartDonePartOutputText, + ContentPartDonePartRefusal, + ContentPartDonePartReasoningText, + ] class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): @@ -1735,19 +1770,6 @@ class OpenAIModerationResponse(BaseLiteLLMOpenAIResponseObject): _hidden_params: dict = PrivateAttr(default_factory=dict) -class OpenAIChatCompletionLogprobsContentTopLogprobs(TypedDict, total=False): - bytes: List - logprob: Required[float] - token: Required[str] - - -class OpenAIChatCompletionLogprobsContent(TypedDict, total=False): - bytes: List - logprob: Required[float] - token: Required[str] - top_logprobs: List[OpenAIChatCompletionLogprobsContentTopLogprobs] - - class OpenAIChatCompletionLogprobs(TypedDict, total=False): content: List[OpenAIChatCompletionLogprobsContent] refusal: List[OpenAIChatCompletionLogprobsContent] From 43826e3cdd8e2b9419fbc9a930bbae17b3b182fa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 Oct 2025 15:55:21 -0700 Subject: [PATCH 22/39] fix(litellm-proxy-extras/utils.py): run resolve_all_migrations after running prisma migrate deploy catch any drift between prisma schema and db Fixes reoccuring issues for users where db columns are missing --- litellm-proxy-extras/litellm_proxy_extras/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index ece2b496bf6..73065b050b7 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -131,7 +131,9 @@ class ProxyExtrasDBManager: ) @staticmethod - def _resolve_all_migrations(migrations_dir: str, schema_path: str): + def _resolve_all_migrations( + migrations_dir: str, schema_path: str, mark_all_applied: bool = True + ): """ 1. Compare the current database state to schema.prisma and generate a migration for the diff. 2. Run prisma migrate deploy to apply any pending migrations. @@ -210,6 +212,8 @@ class ProxyExtrasDBManager: logger.warning("Migration diff application timed out.") # 3. Mark all migrations as applied + if not mark_all_applied: + return migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir) logger.info(f"Resolving {len(migration_names)} migrations") for migration_name in migration_names: @@ -263,6 +267,13 @@ class ProxyExtrasDBManager: logger.info(f"prisma migrate deploy stdout: {result.stdout}") logger.info("prisma migrate deploy completed") + + # Run sanity check to ensure DB matches schema + logger.info("Running post-migration sanity check...") + ProxyExtrasDBManager._resolve_all_migrations( + migrations_dir, schema_path, mark_all_applied=False + ) + logger.info("✅ Post-migration sanity check completed") return True except subprocess.CalledProcessError as e: logger.info(f"prisma db error: {e.stderr}, e: {e.stdout}") From 8661b3aaa684773246050e132032597a23c7f218 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 13:36:36 -0700 Subject: [PATCH 23/39] fix: fix linting error --- litellm/responses/streaming_iterator.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index eda3e6921da..b78913402ff 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -93,24 +93,35 @@ class BaseResponsesAPIStreamingIterator: # Store the completed response if ( openai_responses_api_chunk - and openai_responses_api_chunk.type + and getattr(openai_responses_api_chunk, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED ): self.completed_response = openai_responses_api_chunk # Add cost to usage object if include_cost_in_streaming_usage is True - if litellm.include_cost_in_streaming_usage and self.logging_obj is not None: - response_obj: Optional[ResponsesAPIResponse] = getattr(openai_responses_api_chunk, "response", None) + if ( + litellm.include_cost_in_streaming_usage + and self.logging_obj is not None + ): + response_obj: Optional[ResponsesAPIResponse] = getattr( + openai_responses_api_chunk, "response", None + ) if response_obj: - usage_obj: Optional[ResponseAPIUsage] = getattr(response_obj, "usage", None) + usage_obj: Optional[ResponseAPIUsage] = getattr( + response_obj, "usage", None + ) if usage_obj is not None: try: - cost: Optional[float] = self.logging_obj._response_cost_calculator(result=response_obj) + cost: Optional[float] = ( + self.logging_obj._response_cost_calculator( + result=response_obj + ) + ) if cost is not None: setattr(usage_obj, "cost", cost) except Exception: # If cost calculation fails, continue without cost pass - + self._handle_logging_completed_response() return openai_responses_api_chunk From ee1e9da7b227d6551bb46e6843ba28fdb947b402 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 13:38:28 -0700 Subject: [PATCH 24/39] fix: fix linting errors --- .../streaming_iterator.py | 2 ++ litellm/types/llms/openai.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index e0cbea501f9..d3a8cea3497 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -10,6 +10,7 @@ from litellm.responses.litellm_completion_transformation.transformation import ( from litellm.responses.streaming_iterator import ResponsesAPIStreamingIterator from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( + PART_UNION_TYPES, BaseLiteLLMOpenAIResponseObject, ContentPartAddedEvent, ContentPartDoneEvent, @@ -203,6 +204,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): text = getattr(litellm_complete_object.choices[0].message, "content", "") or "" # type: ignore reasoning_content = getattr(litellm_complete_object.choices[0].message, "reasoning_content", "") or "" # type: ignore + part: Optional[PART_UNION_TYPES] = None if reasoning_content: part = ContentPartDonePartReasoningText( type="reasoning_text", diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 5306dc8bfc2..c9761beeafc 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1059,7 +1059,9 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): parallel_tool_calls: Optional[bool] = None temperature: Optional[float] = None tool_choice: Optional[ToolChoice] = None - tools: Optional[Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]]] = None + tools: Optional[ + Union[List[Tool], List[ResponseFunctionToolCall], List[Dict[str, Any]]] + ] = None top_p: Optional[float] = None max_output_tokens: Optional[int] = None previous_response_id: Optional[str] = None @@ -1231,16 +1233,19 @@ class ContentPartDonePartReasoningText(BaseLiteLLMOpenAIResponseObject): reasoning: str +PART_UNION_TYPES = Union[ + ContentPartDonePartOutputText, + ContentPartDonePartRefusal, + ContentPartDonePartReasoningText, +] + + class ContentPartDoneEvent(BaseLiteLLMOpenAIResponseObject): type: Literal[ResponsesAPIStreamEvents.CONTENT_PART_DONE] item_id: str output_index: int content_index: int - part: Union[ - ContentPartDonePartOutputText, - ContentPartDonePartRefusal, - ContentPartDonePartReasoningText, - ] + part: PART_UNION_TYPES class OutputTextDeltaEvent(BaseLiteLLMOpenAIResponseObject): From ef62682905b61d6dbf3d7ae06d2fe47bb652248b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 13:45:41 -0700 Subject: [PATCH 25/39] fix: fix transformation.py --- .../transformation.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 683f06ee0c7..b060f22d355 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -57,22 +57,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) -> Tuple[Optional[Any], int]: """ Handle raw dict response items from Responses API (e.g., GPT-5 Codex format). - + Args: item: Raw dict response item with 'type' field index: Current choice index - + Returns: Tuple of (Choice object or None, updated index) """ from litellm.types.utils import Choices, Message item_type = item.get("type") - + # Ignore reasoning items for now if item_type == "reasoning": return None, index - + # Handle message items with output_text content if item_type == "message": content_list = item.get("content", []) @@ -83,13 +83,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): response_text = content_item.get("text", "") msg = Message( role=item.get("role", "assistant"), - content=response_text if response_text else "" - ) - choice = Choices( - message=msg, finish_reason="stop", index=index + content=response_text if response_text else "", ) + choice = Choices(message=msg, finish_reason="stop", index=index) return choice, index + 1 - + # Unknown or unsupported type return None, index @@ -294,8 +292,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(item, ResponseReasoningItem): - for content in item.summary: - response_text = getattr(content, "text", "") + for summary_item in item.summary: + response_text = getattr(summary_item, "text", "") reasoning_content = response_text if response_text else "" elif isinstance(item, ResponseOutputMessage): @@ -340,7 +338,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): index += 1 elif isinstance(item, dict): # Handle raw dict responses (e.g., from GPT-5 Codex) - choice, index = self._handle_raw_dict_response_item(item=item, index=index) + choice, index = self._handle_raw_dict_response_item( + item=item, index=index + ) if choice is not None: choices.append(choice) else: From 5f307bf6c1bc25058eee6d98d367db864f5a23fb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 13:47:45 -0700 Subject: [PATCH 26/39] fix: fix linting errors --- .../responses/mcp/mcp_streaming_iterator.py | 368 ++++++++++-------- 1 file changed, 201 insertions(+), 167 deletions(-) diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 7d53452c1c0..8801e561915 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -1,19 +1,10 @@ -from litellm._uuid import uuid -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - Optional, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast from litellm._logging import verbose_logger -from litellm.responses.streaming_iterator import ( - BaseResponsesAPIStreamingIterator, -) +from litellm._uuid import uuid +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( + BaseLiteLLMOpenAIResponseObject, MCPCallArgumentsDeltaEvent, MCPCallArgumentsDoneEvent, MCPCallCompletedEvent, @@ -38,22 +29,24 @@ async def create_mcp_list_tools_events( mcp_tools_with_litellm_proxy: List[ToolParam], user_api_key_auth: Any, base_item_id: str, - pre_processed_mcp_tools: List[Any] + pre_processed_mcp_tools: List[Any], ) -> List[ResponsesAPIStreamingResponse]: """Create MCP discovery events using pre-processed tools from the parent""" - + events: List[ResponsesAPIStreamingResponse] = [] - + try: # Extract MCP server names mcp_servers = [] for tool in mcp_tools_with_litellm_proxy: if isinstance(tool, dict) and "server_url" in tool: server_url = tool.get("server_url") - if isinstance(server_url, str) and server_url.startswith("litellm_proxy/mcp/"): + if isinstance(server_url, str) and server_url.startswith( + "litellm_proxy/mcp/" + ): server_name = server_url.split("/")[-1] mcp_servers.append(server_name) - + # Emit list tools in progress event in_progress_event = MCPListToolsInProgressEvent( type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS, @@ -62,21 +55,21 @@ async def create_mcp_list_tools_events( item_id=base_item_id, ) events.append(in_progress_event) - + # Use the pre-processed MCP tools that were already fetched, filtered, and deduplicated by the parent filtered_mcp_tools = pre_processed_mcp_tools - + # Convert tools to dict format for the event mcp_tools_dict = [] for tool in filtered_mcp_tools: - if hasattr(tool, 'model_dump') and callable(getattr(tool, 'model_dump')): + if hasattr(tool, "model_dump") and callable(getattr(tool, "model_dump")): # Type cast to help mypy understand this is safe after hasattr check mcp_tools_dict.append(cast(Any, tool).model_dump()) - elif hasattr(tool, '__dict__'): + elif hasattr(tool, "__dict__"): mcp_tools_dict.append(tool.__dict__) else: - mcp_tools_dict.append({"name": getattr(tool, 'name', str(tool))}) - + mcp_tools_dict.append({"name": getattr(tool, "name", str(tool))}) + # Emit list tools completed event completed_event = MCPListToolsCompletedEvent( type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED, @@ -85,7 +78,7 @@ async def create_mcp_list_tools_events( item_id=base_item_id, ) events.append(completed_event) - + # Add output_item.done event with the actual tools list (matching OpenAI format) from litellm.types.llms.openai import OutputItemDoneEvent @@ -95,45 +88,50 @@ async def create_mcp_list_tools_events( first_tool = mcp_tools_with_litellm_proxy[0] if isinstance(first_tool, dict): server_label_value = first_tool.get("server_label", "") - server_label = str(server_label_value) if server_label_value is not None else "" - + server_label = ( + str(server_label_value) if server_label_value is not None else "" + ) + # Format tools for OpenAI output_item.done format formatted_tools = [] for tool in filtered_mcp_tools: tool_dict = { - "name": getattr(tool, 'name', 'unknown'), - "description": getattr(tool, 'description', ''), + "name": getattr(tool, "name", "unknown"), + "description": getattr(tool, "description", ""), "annotations": {"read_only": False}, } - + # Add input_schema if available - if hasattr(tool, 'inputSchema'): - tool_dict["input_schema"] = getattr(tool, 'inputSchema') - elif hasattr(tool, 'input_schema'): - tool_dict["input_schema"] = getattr(tool, 'input_schema') - + if hasattr(tool, "inputSchema"): + tool_dict["input_schema"] = getattr(tool, "inputSchema") + elif hasattr(tool, "input_schema"): + tool_dict["input_schema"] = getattr(tool, "input_schema") + formatted_tools.append(tool_dict) - + # Create the output_item.done event with MCP tools list output_item_done_event = OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, - item={ - "id": base_item_id, - "type": "mcp_list_tools", - "server_label": server_label, - "tools": formatted_tools - } + item=BaseLiteLLMOpenAIResponseObject( + **{ + "id": base_item_id, + "type": "mcp_list_tools", + "server_label": server_label, + "tools": formatted_tools, + } + ), ) events.append(output_item_done_event) - + verbose_logger.debug(f"Created {len(events)} MCP discovery events") - + except Exception as e: verbose_logger.error(f"Error creating MCP list tools events: {e}") import traceback + traceback.print_exc() - + # Emit failed event on error failed_event = MCPListToolsFailedEvent( type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED, @@ -142,37 +140,39 @@ async def create_mcp_list_tools_events( item_id=base_item_id, ) events.append(failed_event) - + # Still emit output_item.done event even on failure (with empty tools list) from litellm.types.llms.openai import OutputItemDoneEvent - + output_item_done_event = OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, - item={ - "id": base_item_id, - "type": "mcp_list_tools", - "server_label": "", - "tools": [] - } + item=BaseLiteLLMOpenAIResponseObject( + **{ + "id": base_item_id, + "type": "mcp_list_tools", + "server_label": "", + "tools": [], + } + ), ) events.append(output_item_done_event) - + return events def create_mcp_call_events( - tool_name: str, - tool_call_id: str, + tool_name: str, + tool_call_id: str, arguments: str, result: Optional[str] = None, base_item_id: Optional[str] = None, - sequence_start: int = 1 + sequence_start: int = 1, ) -> List[ResponsesAPIStreamingResponse]: """Create MCP call events following OpenAI's specification""" events: List[ResponsesAPIStreamingResponse] = [] item_id = base_item_id or f"mcp_{uuid.uuid4().hex[:8]}" - + # MCP call in progress event in_progress_event = MCPCallInProgressEvent( type=ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS, @@ -181,7 +181,7 @@ def create_mcp_call_events( item_id=item_id, ) events.append(in_progress_event) - + # MCP call arguments delta event (streaming the arguments) arguments_delta_event = MCPCallArgumentsDeltaEvent( type=ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA, @@ -191,7 +191,7 @@ def create_mcp_call_events( sequence_number=sequence_start + 1, ) events.append(arguments_delta_event) - + # MCP call arguments done event arguments_done_event = MCPCallArgumentsDoneEvent( type=ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE, @@ -201,7 +201,7 @@ def create_mcp_call_events( sequence_number=sequence_start + 2, ) events.append(arguments_done_event) - + # MCP call completed event (or failed if result indicates failure) if result is not None: completed_event = MCPCallCompletedEvent( @@ -211,23 +211,25 @@ def create_mcp_call_events( output_index=0, ) events.append(completed_event) - + # Add output_item.done event with the tool call result from litellm.types.llms.openai import OutputItemDoneEvent - + output_item_done_event = OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, - item={ - "id": item_id, - "type": "mcp_call", - "approval_request_id": f"mcpr_{uuid.uuid4().hex[:8]}", - "arguments": arguments, - "error": None, - "name": tool_name, - "output": result, - "server_label": "litellm" - }, + item=BaseLiteLLMOpenAIResponseObject( + **{ + "id": item_id, + "type": "mcp_call", + "approval_request_id": f"mcpr_{uuid.uuid4().hex[:8]}", + "arguments": arguments, + "error": None, + "name": tool_name, + "output": result, + "server_label": "litellm", + } + ), ) events.append(output_item_done_event) else: @@ -238,7 +240,7 @@ def create_mcp_call_events( output_index=0, ) events.append(failed_event) - + return events @@ -250,51 +252,60 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): 3. Handles tool execution and follow-up calls for auto-execute tools 4. Emits tool execution events in the stream """ - + def __init__( self, base_iterator: Any, # Can be None - will be created internally mcp_events: List[ResponsesAPIStreamingResponse], mcp_tools_with_litellm_proxy: Optional[List[Any]] = None, user_api_key_auth: Any = None, - original_request_params: Optional[Dict[str, Any]] = None + original_request_params: Optional[Dict[str, Any]] = None, ): # MCP setup self.mcp_tools_with_litellm_proxy = mcp_tools_with_litellm_proxy or [] self.user_api_key_auth = user_api_key_auth self.original_request_params = original_request_params or {} self.should_auto_execute = self._should_auto_execute_tools() - + # Streaming state management self.phase = "mcp_discovery" # mcp_discovery -> initial_response -> tool_execution -> follow_up_response -> finished self.finished = False - + # Event queues and generation flags - self.mcp_discovery_events: List[ResponsesAPIStreamingResponse] = mcp_events # Pre-generated MCP discovery events + self.mcp_discovery_events: List[ResponsesAPIStreamingResponse] = ( + mcp_events # Pre-generated MCP discovery events + ) self.tool_execution_events: List[ResponsesAPIStreamingResponse] = [] self.mcp_discovery_generated = True # Events are already generated - self.mcp_events = mcp_events # Store the initial MCP events for backward compatibility - + self.mcp_events = ( + mcp_events # Store the initial MCP events for backward compatibility + ) + # Iterator references - self.base_iterator: Optional[Union[Any, ResponsesAPIResponse]] = base_iterator # Will be created when needed + self.base_iterator: Optional[Union[Any, ResponsesAPIResponse]] = ( + base_iterator # Will be created when needed + ) self.follow_up_iterator: Optional[Any] = None - + # Response collection for tool execution self.collected_response: Optional[ResponsesAPIResponse] = None - + # Set up model metadata (will be updated when we get the real iterator) - self.model = self.original_request_params.get('model', 'unknown') + self.model = self.original_request_params.get("model", "unknown") self.litellm_metadata = {} - self.custom_llm_provider = self.original_request_params.get('custom_llm_provider', None) - + self.custom_llm_provider = self.original_request_params.get( + "custom_llm_provider", None + ) + # Mark as async iterator self.is_async = True - + def _should_auto_execute_tools(self) -> bool: """Check if tools should be auto-executed""" from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) + return LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools( self.mcp_tools_with_litellm_proxy ) @@ -306,45 +317,49 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): """ Phase-based streaming: 1. mcp_discovery - Emit MCP discovery events - 2. initial_response - Stream the first LLM response + 2. initial_response - Stream the first LLM response 3. tool_execution - Emit tool execution events 4. follow_up_response - Stream the follow-up response 5. finished - End iteration """ - + # Phase 1: MCP Discovery Events if self.phase == "mcp_discovery": # Generate MCP discovery events if not already done # MCP discovery events are already generated and available - + # Emit MCP discovery events if self.mcp_discovery_events: return self.mcp_discovery_events.pop(0) - + # All MCP discovery events emitted, move to next phase - verbose_logger.debug("MCP discovery phase complete, transitioning to initial_response") + verbose_logger.debug( + "MCP discovery phase complete, transitioning to initial_response" + ) self.phase = "initial_response" await self._create_initial_response_iterator() # Fall through to process the initial response immediately - + # Phase 2: Initial Response Stream if self.phase == "initial_response": if self.base_iterator: # Check if base_iterator is actually iterable - if hasattr(self.base_iterator, '__anext__'): + if hasattr(self.base_iterator, "__anext__"): try: chunk = await cast(Any, self.base_iterator).__anext__() # type: ignore[attr-defined] - + # If auto-execution is enabled, check for completed responses - if self.should_auto_execute and self._is_response_completed(chunk): + if self.should_auto_execute and self._is_response_completed( + chunk + ): # Collect the response for tool execution - response_obj = getattr(chunk, 'response', None) + response_obj = getattr(chunk, "response", None) if isinstance(response_obj, ResponsesAPIResponse): self.collected_response = response_obj # Move to tool execution phase after emitting this chunk self.phase = "tool_execution" await self._generate_tool_execution_events() - + return chunk except StopAsyncIteration: # Initial response ended, move to next phase @@ -357,24 +372,26 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): else: # base_iterator is not async iterable (likely a ResponsesAPIResponse) # Collect it for tool execution if needed - if self.should_auto_execute and isinstance(self.base_iterator, ResponsesAPIResponse): + if self.should_auto_execute and isinstance( + self.base_iterator, ResponsesAPIResponse + ): self.collected_response = self.base_iterator self.phase = "tool_execution" await self._generate_tool_execution_events() else: self.phase = "finished" raise StopAsyncIteration - + # Phase 3: Tool Execution Events if self.phase == "tool_execution": # Emit any queued tool execution events if self.tool_execution_events: return self.tool_execution_events.pop(0) - + # Move to follow-up response phase self.phase = "follow_up_response" await self._create_follow_up_iterator() - + # Phase 4: Follow-up Response Stream if self.phase == "follow_up_response": if self.follow_up_iterator: @@ -386,20 +403,22 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): else: self.phase = "finished" raise StopAsyncIteration - + # Phase 5: Finished if self.phase == "finished": raise StopAsyncIteration - + # Should not reach here raise StopAsyncIteration - + def _is_response_completed(self, chunk: ResponsesAPIStreamingResponse) -> bool: """Check if this chunk indicates the response is completed""" from litellm.types.llms.openai import ResponsesAPIStreamEvents - return getattr(chunk, 'type', None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED - - + + return ( + getattr(chunk, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + async def _create_initial_response_iterator(self) -> None: """Create the initial response iterator by making the first LLM call""" try: @@ -408,38 +427,45 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Make the initial response API call - but avoid the MCP wrapper params = self.original_request_params.copy() - params['stream'] = True # Ensure streaming - + params["stream"] = True # Ensure streaming + # Use the pre-fetched all_tools from original_request_params (no re-processing needed) params_for_llm = {} for key, value in params.items(): - params_for_llm[key] = value # Copy all params as-is since tools are already processed - - tools_count = len(params_for_llm.get('tools', [])) + params_for_llm[key] = ( + value # Copy all params as-is since tools are already processed + ) + + tools_count = len(params_for_llm.get("tools", [])) verbose_logger.debug(f"Making LLM call with {tools_count} tools") response = await aresponses(**params_for_llm) - + # Set the base iterator - if hasattr(response, '__aiter__') or hasattr(response, '__iter__'): + if hasattr(response, "__aiter__") or hasattr(response, "__iter__"): self.base_iterator = response # Copy metadata from the real iterator - self.model = getattr(response, 'model', self.model) - self.litellm_metadata = getattr(response, 'litellm_metadata', {}) - self.custom_llm_provider = getattr(response, 'custom_llm_provider', self.custom_llm_provider) - verbose_logger.debug(f"Created base iterator: {type(self.base_iterator)}") + self.model = getattr(response, "model", self.model) + self.litellm_metadata = getattr(response, "litellm_metadata", {}) + self.custom_llm_provider = getattr( + response, "custom_llm_provider", self.custom_llm_provider + ) + verbose_logger.debug( + f"Created base iterator: {type(self.base_iterator)}" + ) else: # Non-streaming response - this shouldn't happen but handle it verbose_logger.warning(f"Got non-streaming response: {type(response)}") self.base_iterator = None self.phase = "finished" - + except Exception as e: verbose_logger.error(f"Error creating initial response iterator: {e}") import traceback + traceback.print_exc() self.base_iterator = None self.phase = "finished" - + async def _generate_tool_execution_events(self) -> None: """Generate tool execution events and execute tools""" if not self.collected_response: @@ -447,7 +473,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) - + try: # Extract tool calls from the response if self.collected_response is not None: @@ -456,9 +482,11 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): tool_calls = [] if not tool_calls: return - + for tool_call in tool_calls: - tool_name, tool_arguments, tool_call_id = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) + tool_name, tool_arguments, tool_call_id = ( + LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) + ) if tool_name and tool_call_id: # Create MCP call events for this tool execution call_events = create_mcp_call_events( @@ -467,34 +495,35 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): arguments=tool_arguments or "{}", # JSON string with arguments result=None, # Will be set after execution base_item_id=f"mcp_{uuid.uuid4().hex[:8]}", - sequence_start=len(self.tool_execution_events) + 1 + sequence_start=len(self.tool_execution_events) + 1, ) # Add the in_progress and arguments events (not the completed event yet) self.tool_execution_events.extend(call_events[:-1]) - + # Execute the tools tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( - tool_calls=tool_calls, - user_api_key_auth=self.user_api_key_auth + tool_calls=tool_calls, user_api_key_auth=self.user_api_key_auth ) - + # Create completion events and output_item.done events for tool execution for tool_result in tool_results: tool_call_id = tool_result.get("tool_call_id", "unknown") result_text = tool_result.get("result", "") - + # Find matching tool name and arguments tool_name = "unknown" tool_arguments = "{}" for tool_call in tool_calls: - name, args, call_id = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) + name, args, call_id = ( + LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call) + ) if call_id == tool_call_id: tool_name = name or "unknown" tool_arguments = args or "{}" break - + item_id = f"mcp_{uuid.uuid4().hex[:8]}" - + # Create the completion event completed_event = MCPCallCompletedEvent( type=ResponsesAPIStreamEvents.MCP_CALL_COMPLETED, @@ -503,79 +532,84 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): output_index=0, ) self.tool_execution_events.append(completed_event) - + # Create output_item.done event with the tool call result from litellm.types.llms.openai import OutputItemDoneEvent - + output_item_done_event = OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, output_index=0, - item={ - "id": item_id, - "type": "mcp_call", - "approval_request_id": f"mcpr_{uuid.uuid4().hex[:8]}", - "arguments": tool_arguments, - "error": None, - "name": tool_name, - "output": result_text, - "server_label": "litellm" # or extract from tool config - }, + item=BaseLiteLLMOpenAIResponseObject( + **{ + "id": item_id, + "type": "mcp_call", + "approval_request_id": f"mcpr_{uuid.uuid4().hex[:8]}", + "arguments": tool_arguments, + "error": None, + "name": tool_name, + "output": result_text, + "server_label": "litellm", # or extract from tool config + } + ), ) self.tool_execution_events.append(output_item_done_event) - + # Store tool results for follow-up call self.tool_results = tool_results - + except Exception as e: verbose_logger.error(f"Error in tool execution: {e}") import traceback + traceback.print_exc() self.tool_results = [] - + async def _create_follow_up_iterator(self) -> None: """Create the follow-up response iterator with tool results""" - if not self.collected_response or not hasattr(self, 'tool_results'): + if not self.collected_response or not hasattr(self, "tool_results"): return - + from litellm.responses.main import aresponses from litellm.responses.mcp.litellm_proxy_mcp_handler import ( LiteLLM_Proxy_MCP_Handler, ) - + try: # Create follow-up input if self.collected_response is not None: follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input( response=self.collected_response, # type: ignore[arg-type] tool_results=self.tool_results, - original_input=self.original_request_params.get('input') + original_input=self.original_request_params.get("input"), ) - + # Make follow-up call with streaming follow_up_params = self.original_request_params.copy() - follow_up_params.update({ - 'input': follow_up_input, - 'previous_response_id': self.collected_response.id, # type: ignore[attr-defined] - 'stream': True - }) + follow_up_params.update( + { + "input": follow_up_input, + "previous_response_id": self.collected_response.id, # type: ignore[attr-defined] + "stream": True, + } + ) else: return # Remove tool_choice to avoid forcing more tool calls - follow_up_params.pop('tool_choice', None) - + follow_up_params.pop("tool_choice", None) + follow_up_response = await aresponses(**follow_up_params) - + # Set up the follow-up iterator - if hasattr(follow_up_response, '__aiter__'): + if hasattr(follow_up_response, "__aiter__"): self.follow_up_iterator = follow_up_response - + except Exception as e: verbose_logger.error(f"Error creating follow-up iterator: {e}") import traceback + traceback.print_exc() self.follow_up_iterator = None - def __iter__(self): return self @@ -583,11 +617,11 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # First, emit any queued MCP events if self.mcp_events: # type: ignore[attr-defined] return self.mcp_events.pop(0) # type: ignore[attr-defined] - + # Then delegate to the base iterator if not self.is_async: try: - if self.base_iterator and hasattr(self.base_iterator, '__next__'): + if self.base_iterator and hasattr(self.base_iterator, "__next__"): return next(cast(Any, self.base_iterator)) # type: ignore[arg-type] else: raise StopIteration From 4360e1932220d0aa56a43589b23b35113f9f2761 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 13:50:37 -0700 Subject: [PATCH 27/39] fix: fix indent --- .../mcp_server/mcp_server_manager.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 1d8fa7f2feb..2554e46b9ba 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -355,12 +355,12 @@ class MCPServerManager: ) # Update tool name to server name mapping (for both prefixed and base names) - self.tool_name_to_mcp_server_name_mapping[ - base_tool_name - ] = server_prefix - self.tool_name_to_mcp_server_name_mapping[ - prefixed_tool_name - ] = server_prefix + self.tool_name_to_mcp_server_name_mapping[base_tool_name] = ( + server_prefix + ) + self.tool_name_to_mcp_server_name_mapping[prefixed_tool_name] = ( + server_prefix + ) registered_count += 1 verbose_logger.debug( @@ -1188,26 +1188,26 @@ class MCPServerManager: asyncio.create_task(_call_tool_via_client(client, call_tool_params)) ) - # IMPORTANT: Must await tasks INSIDE the context manager to keep connection alive - try: - mcp_responses = await asyncio.gather(*tasks) - except ( - BlockedPiiEntityError, - GuardrailRaisedException, - HTTPException, - ) as e: - # Re-raise guardrail exceptions to properly fail the MCP call - verbose_logger.error( - f"Guardrail blocked MCP tool call during result check: {str(e)}" - ) - raise e + # IMPORTANT: Must await tasks INSIDE the context manager to keep connection alive + try: + mcp_responses = await asyncio.gather(*tasks) + except ( + BlockedPiiEntityError, + GuardrailRaisedException, + HTTPException, + ) as e: + # Re-raise guardrail exceptions to properly fail the MCP call + verbose_logger.error( + f"Guardrail blocked MCP tool call during result check: {str(e)}" + ) + raise e - # If proxy_logging_obj is None, the tool call result is at index 0 - # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task) - result_index = 1 if proxy_logging_obj else 0 - result = mcp_responses[result_index] + # If proxy_logging_obj is None, the tool call result is at index 0 + # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task) + result_index = 1 if proxy_logging_obj else 0 + result = mcp_responses[result_index] - return cast(CallToolResult, result) + return cast(CallToolResult, result) # For OpenAPI tools, await outside the client context try: From f96f2106f7b31d7e9d138d66d35af1f1608de2d6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 15:32:48 -0700 Subject: [PATCH 28/39] fix(pass_through_endpoints.py): fix check for mapped routes --- .../pass_through_endpoints/pass_through_endpoints.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b6914398da8..9dbe1fc5c6b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -42,6 +42,7 @@ from litellm.passthrough import BasePassthroughUtils from litellm.proxy._types import ( ConfigFieldInfo, ConfigFieldUpdate, + LiteLLMRoutes, PassThroughEndpointResponse, PassThroughGenericEndpoint, ProxyException, @@ -979,7 +980,7 @@ def create_pass_through_route( InitPassThroughEndpointHelpers, ) - if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( + if not not InitPassThroughEndpointHelpers.is_registered_pass_through_route( route=endpoint ): raise HTTPException( @@ -1745,6 +1746,12 @@ class InitPassThroughEndpointHelpers: Returns: bool: True if route is a registered pass-through endpoint, False otherwise """ + + ## CHECK IF MAPPED PASS THROUGH ENDPOINT + for mapped_route in LiteLLMRoutes.mapped_pass_through_routes.value: + if route.startswith(mapped_route): + return True + # Fast path: check if any registered route key contains this path # Keys are in format: "{endpoint_id}:exact:{path}" or "{endpoint_id}:subpath:{path}" # Extract unique paths from keys for quick checking From 090a2c0cfebb68f41aad6f4a23fa9684e78db115 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 15:38:28 -0700 Subject: [PATCH 29/39] fix: remove unused uuid import --- litellm/types/llms/openai.py | 1 - log.txt | 78 ++++++++++++++++++++++++++++ tests/llm_translation/test_openai.py | 16 ------ 3 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 log.txt diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c9761beeafc..aed6bcfb576 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,4 +1,3 @@ -import uuid from enum import Enum from os import PathLike from typing import IO, Any, Iterable, List, Literal, Mapping, Optional, Tuple, Union diff --git a/log.txt b/log.txt new file mode 100644 index 00000000000..a0f2afcdb18 --- /dev/null +++ b/log.txt @@ -0,0 +1,78 @@ +============================= test session starts ============================== +platform darwin -- Python 3.11.4, pytest-7.4.1, pluggy-1.2.0 +rootdir: /Users/krrishdholakia/Documents/litellm +plugins: snapshot-0.9.0, cov-5.0.0, timeout-2.2.0, postgresql-7.0.1, respx-0.21.1, asyncio-0.21.1, langsmith-0.3.4, anyio-4.8.0, mock-3.11.1, Faker-25.9.2 +asyncio: mode=Mode.STRICT +collected 1 item + +tests/llm_translation/test_gemini.py . [100%] + +=============================== warnings summary =============================== +tests/llm_translation/base_llm_unit_tests.py:481 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:481: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:523 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:523: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:601 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:601: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:641 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:641: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:650 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:650: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:694 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:694: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:745 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:745: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:783 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:783: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=6, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:859 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:859: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=4, delay=2) + +tests/llm_translation/base_llm_unit_tests.py:955 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:955: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=4, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:1073 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:1073: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=3, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:1109 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:1109: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=3, delay=1) + +tests/llm_translation/base_llm_unit_tests.py:1232 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/base_llm_unit_tests.py:1232: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=3, delay=1) + +tests/llm_translation/test_gemini.py:36 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/test_gemini.py:36: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=3, delay=2) + +tests/llm_translation/test_gemini.py:510 + /Users/krrishdholakia/Documents/litellm/tests/llm_translation/test_gemini.py:510: PytestUnknownMarkWarning: Unknown pytest.mark.flaky - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html + @pytest.mark.flaky(retries=3, delay=2) + +tests/llm_translation/test_gemini.py::test_gemini_image_generation_async + /Users/krrishdholakia/Library/Python/3.11/lib/python/site-packages/pydantic/main.py:463: UserWarning: Pydantic serializer warnings: + PydanticSerializationUnexpectedValue(Expected 10 fields but got 7: Expected `Message` - serialized value may not be as expected [input_value=Message(content="Here's t...er_specific_fields=None), input_type=Message]) + PydanticSerializationUnexpectedValue(Expected `StreamingChoices` - serialized value may not be as expected [input_value=Choices(finish_reason='st...r_specific_fields=None)), input_type=Choices]) + return self.__pydantic_serializer__.to_python( + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +======================== 1 passed, 16 warnings in 5.86s ======================== diff --git a/tests/llm_translation/test_openai.py b/tests/llm_translation/test_openai.py index 4b63311f1a7..15bb5fec66d 100644 --- a/tests/llm_translation/test_openai.py +++ b/tests/llm_translation/test_openai.py @@ -728,22 +728,6 @@ def test_openai_safety_identifier_parameter_sync(): assert request_body["safety_identifier"] == "user_code_123456" -def test_gpt_5_reasoning(): - litellm._turn_on_debug() - response = litellm.completion( - model="openai/responses/gpt-5-mini", - messages=[ - { - "role": "user", - "content": "Think of the capital of France, and then write it.", - } - ], - reasoning_effort="low", - ) - print("response: ", response) - assert response.choices[0].message.reasoning_content is not None - - def test_gpt_5_reasoning_streaming(): litellm._turn_on_debug() response = litellm.completion( From 8eb58db989ae3d64819347679889485a847c30aa Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 15:42:53 -0700 Subject: [PATCH 30/39] fix: fix linting error --- ui/litellm-dashboard/src/components/networking.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index ce9092ca5ae..713cb1d58ce 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -4442,7 +4442,7 @@ export const getGeneralSettingsCall = async (accessToken: string) => { } }; -export const getPassThroughEndpointsCall = async (accessToken: String, teamId?: string | null) => { +export const getPassThroughEndpointsCall = async (accessToken: string, teamId?: string | null) => { try { let url = proxyBaseUrl ? `${proxyBaseUrl}/config/pass_through_endpoint` : `/config/pass_through_endpoint`; From 6bd722bba4684ce35fb9995cd087dd556e469cd1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 16:09:33 -0700 Subject: [PATCH 31/39] fix: always retain config models --- .../pass_through_endpoints.py | 11 ++-- litellm/proxy/proxy_server.py | 59 +++++++++++-------- .../test_openai_assistants_passthrough.py | 14 +++-- 3 files changed, 52 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 9dbe1fc5c6b..18bab49eb4b 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1819,14 +1819,17 @@ async def initialize_pass_through_endpoints( verbose_proxy_logger.debug("initializing pass through endpoints") from litellm.proxy._types import CommonProxyErrors, LiteLLMRoutes - from litellm.proxy.proxy_server import app, general_settings, premium_user + from litellm.proxy.proxy_server import ( + app, + config_passthrough_endpoints, + premium_user, + ) ## get combined pass-through endpoints from db + config - config_pass_through_endpoints = general_settings.get("pass_through_endpoints") combined_pass_through_endpoints: List[Union[Dict, PassThroughGenericEndpoint]] - if config_pass_through_endpoints is not None: + if config_passthrough_endpoints is not None: combined_pass_through_endpoints = _get_combined_pass_through_endpoints( # type: ignore - pass_through_endpoints, config_pass_through_endpoints + pass_through_endpoints, config_passthrough_endpoints ) else: combined_pass_through_endpoints = pass_through_endpoints # type: ignore diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index be0f7d6be08..8f5a046be03 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -259,9 +259,7 @@ from litellm.proxy.management_endpoints.customer_endpoints import ( from litellm.proxy.management_endpoints.internal_user_endpoints import ( router as internal_user_router, ) -from litellm.proxy.management_endpoints.internal_user_endpoints import ( - user_update, -) +from litellm.proxy.management_endpoints.internal_user_endpoints import user_update from litellm.proxy.management_endpoints.key_management_endpoints import ( delete_verification_tokens, duration_in_seconds, @@ -308,9 +306,7 @@ from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMi from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) -from litellm.proxy.openai_files_endpoints.files_endpoints import ( - set_files_config, -) +from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( passthrough_endpoint_router, ) @@ -580,7 +576,7 @@ async def _initialize_shared_aiohttp_session(): ttl_dns_cache=AIOHTTP_TTL_DNS_CACHE, enable_cleanup_closed=True, ) - + session = ClientSession(connector=connector) verbose_proxy_logger.info( f"SESSION REUSE: Created shared aiohttp session for connection pooling (ID: {id(session)})" @@ -723,7 +719,7 @@ async def proxy_startup_event(app: FastAPI): verbose_proxy_logger.info("SESSION REUSE: Closed shared aiohttp session") except Exception as e: verbose_proxy_logger.error(f"Error closing shared aiohttp session: {e}") - + await proxy_shutdown_event() @@ -995,13 +991,16 @@ experimental = False llm_router: Optional[Router] = None llm_model_list: Optional[list] = None general_settings: dict = {} +config_passthrough_endpoints: Optional[List[Dict[str, Any]]] = None callback_settings: dict = {} log_file = "api_log.json" worker_config = None master_key: Optional[str] = None otel_logging = False prisma_client: Optional[PrismaClient] = None -shared_aiohttp_session: Optional["ClientSession"] = None # Global shared session for connection reuse +shared_aiohttp_session: Optional["ClientSession"] = ( + None # Global shared session for connection reuse +) user_api_key_cache = DualCache( default_in_memory_ttl=UserAPIKeyCacheTTLEnum.in_memory_cache_ttl.value ) @@ -1385,31 +1384,33 @@ async def update_cache( # noqa: PLR0915 """ if tags is None or response_cost is None: return - + try: for tag_name in tags: if not tag_name or not isinstance(tag_name, str): continue - + cache_key = f"tag:{tag_name}" # Fetch the existing tag object from cache - existing_tag_obj = await user_api_key_cache.async_get_cache(key=cache_key) + existing_tag_obj = await user_api_key_cache.async_get_cache( + key=cache_key + ) if existing_tag_obj is None: # do nothing if tag not in api key cache continue - + verbose_proxy_logger.debug( f"_update_tag_cache: existing spend for tag={tag_name}: {existing_tag_obj}; response_cost: {response_cost}" ) - + if isinstance(existing_tag_obj, dict): existing_spend = existing_tag_obj.get("spend", 0) or 0 else: existing_spend = getattr(existing_tag_obj, "spend", 0) or 0 - + # Calculate the new cost by adding the existing cost and response_cost new_spend = existing_spend + response_cost - + # Update the spend column for the given tag if isinstance(existing_tag_obj, dict): existing_tag_obj["spend"] = new_spend @@ -1482,6 +1483,7 @@ async def _run_background_health_check(): from litellm.proxy.health_check_utils.shared_health_check_manager import ( SharedHealthCheckManager, ) + shared_health_manager = SharedHealthCheckManager( redis_cache=redis_usage_cache, health_check_ttl=DEFAULT_SHARED_HEALTH_CHECK_TTL, @@ -1502,16 +1504,21 @@ async def _run_background_health_check(): # Use shared health check if available, otherwise fall back to direct health check # Convert health_check_details to bool for perform_shared_health_check (defaults to True if None) - details_bool = health_check_details if health_check_details is not None else True - + details_bool = ( + health_check_details if health_check_details is not None else True + ) + if shared_health_manager is not None: try: - healthy_endpoints, unhealthy_endpoints = await shared_health_manager.perform_shared_health_check( - model_list=_llm_model_list, details=details_bool + healthy_endpoints, unhealthy_endpoints = ( + await shared_health_manager.perform_shared_health_check( + model_list=_llm_model_list, details=details_bool + ) ) except Exception as e: verbose_proxy_logger.error( - "Error in shared health check, falling back to direct health check: %s", str(e) + "Error in shared health check, falling back to direct health check: %s", + str(e), ) healthy_endpoints, unhealthy_endpoints = await perform_health_check( model_list=_llm_model_list, details=health_check_details @@ -1879,7 +1886,7 @@ class ProxyConfig: """ Load config values into proxy global state """ - global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings, proxy_batch_polling_interval + global master_key, user_config_file_path, otel_logging, user_custom_auth, user_custom_auth_path, user_custom_key_generate, user_custom_sso, user_custom_ui_sso_sign_in_handler, use_background_health_checks, use_shared_health_check, health_check_interval, use_queue, proxy_budget_rescheduler_max_time, proxy_budget_rescheduler_min_time, ui_access_mode, litellm_master_key_hash, proxy_batch_write_at, disable_spend_logs, prompt_injection_detection_obj, redis_usage_cache, store_model_in_db, premium_user, open_telemetry_logger, health_check_details, callback_settings, proxy_batch_polling_interval, config_passthrough_endpoints config: dict = await self.get_config(config_file_path=config_file_path) @@ -2234,9 +2241,13 @@ class ProxyConfig: ## pass through endpoints if general_settings.get("pass_through_endpoints", None) is not None: + config_passthrough_endpoints = general_settings[ + "pass_through_endpoints" + ] await initialize_pass_through_endpoints( pass_through_endpoints=general_settings["pass_through_endpoints"] ) + ## ADMIN UI ACCESS ## ui_access_mode = general_settings.get( "ui_access_mode", "all" @@ -3055,7 +3066,9 @@ class ProxyConfig: return current_config # For dictionary values, update only non-none values - if isinstance(current_config[param_name], dict) and isinstance(db_param_value, dict): + if isinstance(current_config[param_name], dict) and isinstance( + db_param_value, dict + ): _deep_merge_dicts(current_config[param_name], db_param_value) else: # Non-dict or mismatched types: DB value replaces config (unchanged behavior) diff --git a/tests/pass_through_tests/test_openai_assistants_passthrough.py b/tests/pass_through_tests/test_openai_assistants_passthrough.py index e5783877ec0..28568005fd6 100644 --- a/tests/pass_through_tests/test_openai_assistants_passthrough.py +++ b/tests/pass_through_tests/test_openai_assistants_passthrough.py @@ -9,9 +9,12 @@ from openai import AssistantEventHandler client = openai.OpenAI(base_url="http://0.0.0.0:4000/openai", api_key="sk-1234") + def test_pass_through_file_operations(): # Create a temporary file - with tempfile.NamedTemporaryFile(mode='w+', suffix='.txt', delete=False) as temp_file: + with tempfile.NamedTemporaryFile( + mode="w+", suffix=".txt", delete=False + ) as temp_file: temp_file.write("This is a test file for the OpenAI Assistants API.") temp_file.flush() @@ -26,6 +29,7 @@ def test_pass_through_file_operations(): delete_file = client.files.delete(file.id) print("file deleted", delete_file) + def test_openai_assistants_e2e_operations(): assistant = client.beta.assistants.create( name="Math Tutor", @@ -98,13 +102,13 @@ def test_openai_assistants_e2e_operations_stream(): stream.until_done() - def test_azure_openai_assistants_e2e_operations_stream(): from openai import AzureOpenAI + client = AzureOpenAI( - base_url="http://0.0.0.0:4000/azure-config-passthrough/openai", + base_url="http://0.0.0.0:4000/azure-config-passthrough/openai", api_key="sk-1234", - api_version="2025-01-01-preview" + api_version="2025-01-01-preview", ) assistant = client.beta.assistants.create( name="Math Tutor", @@ -134,4 +138,4 @@ def test_azure_openai_assistants_e2e_operations_stream(): instructions="Please address the user as Jane Doe. The user has a premium account.", event_handler=EventHandler(), ) as stream: - stream.until_done() \ No newline at end of file + stream.until_done() From a7456ab21e036a75bf8fff92e89b63185ece2ee7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 16:18:37 -0700 Subject: [PATCH 32/39] refactor: refactor to cut down large functions --- .../mcp_server/mcp_server_manager.py | 166 +++++++++++------- .../management_endpoints/team_endpoints.py | 70 ++++++-- 2 files changed, 158 insertions(+), 78 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 2554e46b9ba..80ae8217d7a 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1058,6 +1058,103 @@ class MCPServerManager: ) ) + async def _call_regular_mcp_tool( + self, + mcp_server: MCPServer, + original_tool_name: str, + arguments: Dict[str, Any], + tasks: List, + mcp_auth_header: Optional[str], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], + oauth2_headers: Optional[Dict[str, str]], + raw_headers: Optional[Dict[str, str]], + proxy_logging_obj: Optional[ProxyLogging], + ) -> CallToolResult: + """ + Call a regular MCP tool using the MCP client. + + Args: + mcp_server: The MCP server configuration + original_tool_name: The original tool name (without prefix) + arguments: Tool arguments + tasks: List of async tasks to append to (for during hooks) + mcp_auth_header: MCP auth header (deprecated) + mcp_server_auth_headers: Optional dict of server-specific auth headers + oauth2_headers: Optional OAuth2 headers + raw_headers: Optional raw headers from the request + proxy_logging_obj: Optional ProxyLogging object for hook integration + + Returns: + CallToolResult from the MCP server + + Raises: + BlockedPiiEntityError: If PII is blocked by guardrails + GuardrailRaisedException: If guardrails block the call + HTTPException: If an HTTP error occurs + """ + # Get server-specific auth header if available + server_auth_header: Optional[Union[Dict[str, str], str]] = None + if mcp_server_auth_headers and mcp_server.alias: + server_auth_header = mcp_server_auth_headers.get(mcp_server.alias) + elif mcp_server_auth_headers and mcp_server.server_name: + server_auth_header = mcp_server_auth_headers.get(mcp_server.server_name) + + # Fall back to deprecated mcp_auth_header if no server-specific header found + if server_auth_header is None: + server_auth_header = mcp_auth_header + + # oauth2 headers + extra_headers: Optional[Dict[str, str]] = None + if mcp_server.auth_type == MCPAuth.oauth2: + extra_headers = oauth2_headers + + if mcp_server.extra_headers and raw_headers: + if extra_headers is None: + extra_headers = {} + for header in mcp_server.extra_headers: + if header in raw_headers: + extra_headers[header] = raw_headers[header] + + client = self._create_mcp_client( + server=mcp_server, + mcp_auth_header=server_auth_header, + extra_headers=extra_headers, + ) + + call_tool_params = MCPCallToolRequestParams( + name=original_tool_name, + arguments=arguments, + ) + + async def _call_tool_via_client(client, params): + async with client: + return await client.call_tool(params) + + tasks.append( + asyncio.create_task(_call_tool_via_client(client, call_tool_params)) + ) + + # IMPORTANT: Must await tasks INSIDE the context manager to keep connection alive + try: + mcp_responses = await asyncio.gather(*tasks) + except ( + BlockedPiiEntityError, + GuardrailRaisedException, + HTTPException, + ) as e: + # Re-raise guardrail exceptions to properly fail the MCP call + verbose_logger.error( + f"Guardrail blocked MCP tool call during result check: {str(e)}" + ) + raise e + + # If proxy_logging_obj is None, the tool call result is at index 0 + # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task) + result_index = 1 if proxy_logging_obj else 0 + result = mcp_responses[result_index] + + return cast(CallToolResult, result) + async def call_tool( self, name: str, @@ -1146,69 +1243,18 @@ class MCPServerManager: ) else: # For regular MCP servers, use the MCP client - # Get server-specific auth header if available - server_auth_header: Optional[Union[Dict[str, str], str]] = None - if mcp_server_auth_headers and mcp_server.alias: - server_auth_header = mcp_server_auth_headers.get(mcp_server.alias) - elif mcp_server_auth_headers and mcp_server.server_name: - server_auth_header = mcp_server_auth_headers.get(mcp_server.server_name) - - # Fall back to deprecated mcp_auth_header if no server-specific header found - if server_auth_header is None: - server_auth_header = mcp_auth_header - - # oauth2 headers - extra_headers: Optional[Dict[str, str]] = None - if mcp_server.auth_type == MCPAuth.oauth2: - extra_headers = oauth2_headers - - if mcp_server.extra_headers and raw_headers: - if extra_headers is None: - extra_headers = {} - for header in mcp_server.extra_headers: - if header in raw_headers: - extra_headers[header] = raw_headers[header] - - client = self._create_mcp_client( - server=mcp_server, - mcp_auth_header=server_auth_header, - extra_headers=extra_headers, - ) - - call_tool_params = MCPCallToolRequestParams( - name=original_tool_name, + return await self._call_regular_mcp_tool( + mcp_server=mcp_server, + original_tool_name=original_tool_name, arguments=arguments, + tasks=tasks, + mcp_auth_header=mcp_auth_header, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + proxy_logging_obj=proxy_logging_obj, ) - async def _call_tool_via_client(client, params): - async with client: - return await client.call_tool(params) - - tasks.append( - asyncio.create_task(_call_tool_via_client(client, call_tool_params)) - ) - - # IMPORTANT: Must await tasks INSIDE the context manager to keep connection alive - try: - mcp_responses = await asyncio.gather(*tasks) - except ( - BlockedPiiEntityError, - GuardrailRaisedException, - HTTPException, - ) as e: - # Re-raise guardrail exceptions to properly fail the MCP call - verbose_logger.error( - f"Guardrail blocked MCP tool call during result check: {str(e)}" - ) - raise e - - # If proxy_logging_obj is None, the tool call result is at index 0 - # If proxy_logging_obj is not None, the tool call result is at index 1 (after the during hook task) - result_index = 1 if proxy_logging_obj else 0 - result = mcp_responses[result_index] - - return cast(CallToolResult, result) - # For OpenAPI tools, await outside the client context try: mcp_responses = await asyncio.gather(*tasks) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 0d02007b8b1..94e0fd3d442 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -627,6 +627,54 @@ async def _update_model_table( return _model_id +async def fetch_and_validate_organization( + organization_id: str, + existing_team_row: Any, + llm_router: Optional[Router], + prisma_client: Any, +) -> Any: + """ + Fetch and validate an organization for team update operations. + + Args: + organization_id: The organization ID to fetch + existing_team_row: The existing team row being updated + llm_router: The LLM router instance + prisma_client: The Prisma database client + + Returns: + The organization row from the database + + Raises: + HTTPException: If llm_router is None, organization not found, or validation fails + """ + if llm_router is None: + raise HTTPException( + status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value} + ) + + organization_row = await prisma_client.db.litellm_organizationtable.find_unique( + where={"organization_id": organization_id}, + include={"litellm_budget_table": True, "users": True}, + ) + + if organization_row is None: + raise HTTPException( + status_code=404, + detail={ + "error": f"Organization not found, passed organization_id={organization_id}" + }, + ) + + validate_team_org_change( + team=LiteLLM_TeamTable(**existing_team_row.model_dump()), + organization=LiteLLM_OrganizationTable(**organization_row.model_dump()), + llm_router=llm_router, + ) + + return organization_row + + def validate_team_org_change( team: LiteLLM_TeamTable, organization: LiteLLM_OrganizationTable, llm_router: Router ) -> bool: @@ -817,25 +865,11 @@ async def update_team( if ( data.organization_id is not None and len(data.organization_id) > 0 ): # allow unsetting the organization_id - if llm_router is None: - raise HTTPException( - status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value} - ) - organization_row = await prisma_client.db.litellm_organizationtable.find_unique( - where={"organization_id": data.organization_id}, - include={"litellm_budget_table": True, "users": True}, - ) - if organization_row is None: - raise HTTPException( - status_code=404, - detail={ - "error": f"Organization not found, passed organization_id={data.organization_id}" - }, - ) - validate_team_org_change( - team=LiteLLM_TeamTable(**existing_team_row.model_dump()), - organization=LiteLLM_OrganizationTable(**organization_row.model_dump()), + await fetch_and_validate_organization( + organization_id=data.organization_id, + existing_team_row=existing_team_row, llm_router=llm_router, + prisma_client=prisma_client, ) elif data.organization_id is not None and len(data.organization_id) == 0: # unsetting the organization_id From 23870cbda03562e8d40572ea02542bd95cec72ab Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 16:38:32 -0700 Subject: [PATCH 33/39] fix(pass_through_endpoints.py): fix typo --- .../proxy/pass_through_endpoints/pass_through_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 18bab49eb4b..e00c4727ebb 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -980,7 +980,7 @@ def create_pass_through_route( InitPassThroughEndpointHelpers, ) - if not not InitPassThroughEndpointHelpers.is_registered_pass_through_route( + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( route=endpoint ): raise HTTPException( @@ -1760,7 +1760,6 @@ class InitPassThroughEndpointHelpers: if len(parts) == 3: route_type = parts[1] registered_path = parts[2] - if route_type == "exact" and route == registered_path: return True elif route_type == "subpath": @@ -1827,6 +1826,7 @@ async def initialize_pass_through_endpoints( ## get combined pass-through endpoints from db + config combined_pass_through_endpoints: List[Union[Dict, PassThroughGenericEndpoint]] + if config_passthrough_endpoints is not None: combined_pass_through_endpoints = _get_combined_pass_through_endpoints( # type: ignore pass_through_endpoints, config_passthrough_endpoints From 2266b272aab6f19151adb721ca4d06cde7336756 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 16:41:03 -0700 Subject: [PATCH 34/39] fix(proxy/_types.py): fix default to be none --- litellm/proxy/_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4a53e2bbb4f..41b36d46c75 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -776,7 +776,7 @@ class KeyRequestBase(GenerateRequestBase): tags: Optional[List[str]] = None enforced_params: Optional[List[str]] = None allowed_routes: Optional[list] = [] - allowed_passthrough_routes: Optional[list] = [] + allowed_passthrough_routes: Optional[list] = None rpm_limit_type: Optional[ Literal["guaranteed_throughput", "best_effort_throughput"] ] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm From a71329fbe1677cf913ef12a4d49b9e601bd94647 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 16:42:22 -0700 Subject: [PATCH 35/39] fix: fix linting errors --- .../litellm_completion_transformation/streaming_iterator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index d3a8cea3497..a4fb2d96032 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -119,8 +119,6 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): response_created_event_data["truncation"] = self.responses_api_request[ "truncation" ] - if "usage" in self.responses_api_request: - response_created_event_data["usage"] = self.responses_api_request["usage"] if "user" in self.responses_api_request: response_created_event_data["user"] = self.responses_api_request["user"] if "metadata" in self.responses_api_request: From 3ee3cb945dd2334828205b2cfadb5fa27a97f551 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 16:45:55 -0700 Subject: [PATCH 36/39] fix(team_endpoints.py): document new param --- litellm/proxy/management_endpoints/team_endpoints.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 94e0fd3d442..f2d2cef3d6f 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -319,6 +319,7 @@ async def new_team( # noqa: PLR0915 - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" - prompts: Optional[List[str]] - List of allowed prompts for the team. If specified, the team will only be able to use these specific prompts. + - allowed_passthrough_routes: Optional[List[str]] - List of allowed pass through routes for the team. Returns: @@ -809,6 +810,7 @@ async def update_team( - team_member_rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for individual team members. - team_member_tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for individual team members. - team_member_key_duration: Optional[str] - The duration for a team member's key. e.g. "1d", "1w", "1mo" + - allowed_passthrough_routes: Optional[List[str]] - List of allowed pass through routes for the team. Example - update team TPM Limit ``` From 78e2274381f8cf8dd61900cb3ba7c85c607a5173 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 Oct 2025 17:04:56 -0700 Subject: [PATCH 37/39] fix(pass_through_endpoints.py): use path instead of endpoint includes the mapped route --- .../proxy/pass_through_endpoints/pass_through_endpoints.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e00c4727ebb..478966d61b3 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -975,13 +975,14 @@ def create_pass_through_route( ] = None, # if pass-through endpoint is a streaming request subpath: str = "", # captures sub-paths when include_subpath=True ): - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( InitPassThroughEndpointHelpers, ) + path = request.url.path + if not InitPassThroughEndpointHelpers.is_registered_pass_through_route( - route=endpoint + route=path ): raise HTTPException( status_code=404, @@ -990,7 +991,7 @@ def create_pass_through_route( passthrough_params = ( InitPassThroughEndpointHelpers.get_registered_pass_through_route( - route=endpoint + route=path ) ) target_params = { From 194943604716352e8a4d7a3ad220bd43b82cdae1 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 12 Oct 2025 21:57:55 -0700 Subject: [PATCH 38/39] test: update test --- .../test_reasoning_content_transformation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py index d1926bbc93f..020b5de0a2a 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_reasoning_content_transformation.py @@ -281,6 +281,7 @@ def test_streaming_chunk_id_raw(): ) iterator = LiteLLMCompletionStreamingIterator( + model="test-model", litellm_custom_stream_wrapper=AsyncMock(), request_input="Test input", responses_api_request={}, From 0ffc81f010b8bf5402fd7360fc123ddafade2664 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sun, 12 Oct 2025 22:01:16 -0700 Subject: [PATCH 39/39] fix: fix reformatting --- litellm/types/llms/openai.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index aed6bcfb576..56971b91e5d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -43,12 +43,7 @@ from openai.types.responses.response import ( # Handle OpenAI SDK version compatibility for Text type try: - # fmt: off - from openai.types.responses.response_create_params import ( - Text as ResponseText, # type: ignore[attr-defined] - ) - - # fmt: on + from openai.types.responses.response_create_params import ( Text as ResponseText ) # type: ignore[attr-defined] # fmt: skip # isort: skip except (ImportError, AttributeError): # Fall back to the concrete config type available in all SDK versions from openai.types.responses.response_text_config_param import (