From 943e036851352e2fba07e2fff4ce64834e4b4055 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 20 Mar 2025 20:11:49 -0700 Subject: [PATCH 1/3] feat(pass_through_endpoints.py): support returning api-base on pass-through endpoints Make it easier to debug what the api base sent to provider was --- litellm/proxy/common_request_processing.py | 4 +++- .../pass_through_endpoints.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 7f131efb043..fcc13509ceb 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -57,7 +57,9 @@ class ProxyBaseLLMRequestProcessing: "x-litellm-call-id": call_id, "x-litellm-model-id": model_id, "x-litellm-cache-key": cache_key, - "x-litellm-model-api-base": api_base, + "x-litellm-model-api-base": ( + api_base.split("?")[0] if api_base else None + ), # don't include query params, risk of leaking sensitive info "x-litellm-version": version, "x-litellm-model-region": model_region, "x-litellm-response-cost": str(response_cost), diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index b13d614678a..63398e553ba 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -284,7 +284,9 @@ class HttpPassThroughEndpointHelpers: @staticmethod def get_response_headers( - headers: httpx.Headers, litellm_call_id: Optional[str] = None + headers: httpx.Headers, + litellm_call_id: Optional[str] = None, + custom_headers: Optional[dict] = None, ) -> dict: excluded_headers = {"transfer-encoding", "content-encoding"} @@ -295,6 +297,8 @@ class HttpPassThroughEndpointHelpers: } if litellm_call_id: return_headers["x-litellm-call-id"] = litellm_call_id + if custom_headers: + return_headers.update(custom_headers) return return_headers @@ -596,12 +600,20 @@ async def pass_through_request( # noqa: PLR0915 ) ) + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=litellm_call_id, + model_id=None, + cache_key=None, + api_base=str(url._uri_reference), + ) + return Response( content=content, status_code=response.status_code, headers=HttpPassThroughEndpointHelpers.get_response_headers( headers=response.headers, - litellm_call_id=litellm_call_id, + custom_headers=custom_headers, ), ) except Exception as e: From 532af66bbdb2a63178c7c5fe391beef424d2d130 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 20 Mar 2025 20:19:52 -0700 Subject: [PATCH 2/3] feat(pass_through_endpoints.py): return api base on pass-through exception enables easy debugging on backend api errors --- .../pass_through_endpoints.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 63398e553ba..a13b0dc216e 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -1,6 +1,7 @@ import ast import asyncio import json +import uuid from base64 import b64encode from datetime import datetime from typing import Dict, List, Optional, Union @@ -369,8 +370,9 @@ async def pass_through_request( # noqa: PLR0915 query_params: Optional[dict] = None, stream: Optional[bool] = None, ): + litellm_call_id = str(uuid.uuid4()) + url: Optional[httpx.URL] = None try: - import uuid from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.proxy_server import proxy_logging_obj @@ -420,8 +422,6 @@ async def pass_through_request( # noqa: PLR0915 ) async_client = async_client_obj.client - litellm_call_id = str(uuid.uuid4()) - # create logging object start_time = datetime.now() logging_obj = Logging( @@ -600,6 +600,7 @@ async def pass_through_request( # noqa: PLR0915 ) ) + ## CUSTOM HEADERS - `x-litellm-*` custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, call_id=litellm_call_id, @@ -617,6 +618,13 @@ async def pass_through_request( # noqa: PLR0915 ), ) except Exception as e: + custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( + user_api_key_dict=user_api_key_dict, + call_id=litellm_call_id, + model_id=None, + cache_key=None, + api_base=str(url._uri_reference) if url else None, + ) verbose_proxy_logger.exception( "litellm.proxy.proxy_server.pass_through_endpoint(): Exception occured - {}".format( str(e) @@ -628,6 +636,7 @@ async def pass_through_request( # noqa: PLR0915 type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + headers=custom_headers, ) else: error_msg = f"{str(e)}" @@ -636,6 +645,7 @@ async def pass_through_request( # noqa: PLR0915 type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), + headers=custom_headers, ) From 0e0925fccf426210a0061e919c798d3aab99e264 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 20 Mar 2025 20:28:03 -0700 Subject: [PATCH 3/3] fix(team_endpoints.py): consistently return 404 if team not found in DB Fixes response on /team/delete --- litellm/proxy/management_endpoints/team_endpoints.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 1994e27ecf5..f5bcc6ba11c 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -470,7 +470,7 @@ async def update_team( if existing_team_row is None: raise HTTPException( - status_code=400, + status_code=404, detail={"error": f"Team not found, passed team_id={data.team_id}"}, ) @@ -1137,14 +1137,16 @@ async def delete_team( team_rows: List[LiteLLM_TeamTable] = [] for team_id in data.team_ids: try: - team_row_base: BaseModel = ( + team_row_base: Optional[BaseModel] = ( await prisma_client.db.litellm_teamtable.find_unique( where={"team_id": team_id} ) ) + if team_row_base is None: + raise Exception except Exception: raise HTTPException( - status_code=400, + status_code=404, detail={"error": f"Team not found, passed team_id={team_id}"}, ) team_row_pydantic = LiteLLM_TeamTable(**team_row_base.model_dump())