From 6f2b753a5de651b2ba9e826f17ce8a5949716b10 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Oct 2025 12:51:41 -0700 Subject: [PATCH 1/5] feat(llm_passthrough_endpoints.py): add router model support on azure passthrough --- ...odel_prices_and_context_window_backup.json | 301 ++++++++++++++++++ litellm/proxy/_new_secret_config.yaml | 1 + .../llm_passthrough_endpoints.py | 195 +++++++++--- litellm/proxy/utils.py | 4 +- 4 files changed, 461 insertions(+), 40 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7ed04dc79c9..fd09aca4c1a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22173,6 +22173,307 @@ "supports_tool_choice": true, "supports_vision": false }, + "watsonx/bigscience/mt0-xxl-13b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/core42/jais-13b-chat": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/google/flan-t5-xl-3b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.00025, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-13b-chat-v2": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-13b-instruct-v2": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-3-3-8b-instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/ibm/granite-4-h-small": { + "max_tokens": 20480, + "max_input_tokens": 20480, + "max_output_tokens": 20480, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.0025, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/ibm/granite-guardian-3-2-2b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-guardian-3-3-8b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-ttm-1024-96-r2": { + "max_tokens": 512, + "max_input_tokens": 512, + "max_output_tokens": 512, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.000625, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-ttm-1536-96-r2": { + "max_tokens": 512, + "max_input_tokens": 512, + "max_output_tokens": 512, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.000625, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-ttm-512-96-r2": { + "max_tokens": 512, + "max_input_tokens": 512, + "max_output_tokens": 512, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.000625, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-vision-3-2-2b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": true + }, + "watsonx/meta-llama/llama-3-2-11b-vision-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "watsonx/meta-llama/llama-3-2-1b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-3-2-3b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-3-2-90b-vision-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.008, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "watsonx/meta-llama/llama-3-3-70b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-4-maverick-17b": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-guard-3-11b-vision": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": true + }, + "watsonx/mistralai/mistral-medium-2505": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00225, + "output_cost_per_token": 0.00675, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/mistralai/mistral-small-2503": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 0.0002, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/mistralai/pixtral-12b-2409": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.00015, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": true + }, + "watsonx/openai/gpt-oss-120b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.004, + "output_cost_per_token": 0.016, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/sdaia/allam-1-13b-instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "whisper-1": { "input_cost_per_second": 0.0001, "litellm_provider": "openai", diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b7b30d36f99..8c929fac8d4 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -16,6 +16,7 @@ model_list: api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" api_key: dummy + # mcp_servers: # github_mcp: # url: "https://api.githubcopilot.com/mcp" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 8aa3b90d954..849eac17f07 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -21,9 +21,7 @@ from litellm.constants import BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.auth.user_api_key_auth import ( - user_api_key_auth, -) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, get_form_data, @@ -31,6 +29,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + HttpPassThroughEndpointHelpers, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -57,7 +56,9 @@ def create_request_copy(request: Request): } -def is_passthrough_request_using_router_model(request_body: dict, llm_router: Optional[litellm.Router]) -> bool: +def is_passthrough_request_using_router_model( + request_body: dict, llm_router: Optional[litellm.Router] +) -> bool: """ Returns True if the model is in the llm_router model names """ @@ -93,12 +94,16 @@ async def llm_passthrough_factory_proxy_route( model=None, ) if provider_config is None: - raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} not found") + raise HTTPException( + status_code=404, detail=f"Provider {custom_llm_provider} not found" + ) base_target_url = provider_config.get_api_base() if base_target_url is None: - raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} api base not found") + raise HTTPException( + status_code=404, detail=f"Provider {custom_llm_provider} api base not found" + ) encoded_endpoint = httpx.URL(endpoint).path @@ -177,11 +182,17 @@ async def gemini_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio) """ ## CHECK FOR LITELLM API KEY IN THE QUERY PARAMS - ?..key=LITELLM_API_KEY - google_ai_studio_api_key = request.query_params.get("key") or request.headers.get("x-goog-api-key") + google_ai_studio_api_key = request.query_params.get("key") or request.headers.get( + "x-goog-api-key" + ) - user_api_key_dict = await user_api_key_auth(request=request, api_key=f"Bearer {google_ai_studio_api_key}") + user_api_key_dict = await user_api_key_auth( + request=request, api_key=f"Bearer {google_ai_studio_api_key}" + ) - base_target_url = os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com" + base_target_url = ( + os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com" + ) encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -293,13 +304,12 @@ async def vllm_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/vllm) """ - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( - HttpPassThroughEndpointHelpers, - ) from litellm.proxy.proxy_server import llm_router request_body = await get_request_body(request) - is_router_model = is_passthrough_request_using_router_model(request_body, llm_router) + is_router_model = is_passthrough_request_using_router_model( + request_body, llm_router + ) is_streaming_request = is_passthrough_request_streaming(request_body) if is_router_model and llm_router: result = cast( @@ -314,7 +324,11 @@ async def vllm_proxy_route( content=None, data=None, files=None, - json=(request_body if request.headers.get("content-type") == "application/json" else None), + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None + ), params=None, headers=None, cookies=None, @@ -492,7 +506,9 @@ async def handle_bedrock_count_tokens( # Extract model from request body model = request_body.get("model") if not model: - raise HTTPException(status_code=400, detail={"error": "Model is required in request body"}) + raise HTTPException( + status_code=400, detail={"error": "Model is required in request body"} + ) # Get model parameters from router litellm_params = {"user_api_key_dict": user_api_key_dict} @@ -531,7 +547,9 @@ async def handle_bedrock_count_tokens( raise except Exception as e: verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {str(e)}") - raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {str(e)}"}) + raise HTTPException( + status_code=500, detail={"error": f"CountTokens processing error: {str(e)}"} + ) async def bedrock_llm_proxy_route( @@ -583,7 +601,8 @@ async def bedrock_llm_proxy_route( raise HTTPException( status_code=400, detail={ - "error": "Model missing from endpoint. Expected format: /model//. Got: " + endpoint, + "error": "Model missing from endpoint. Expected format: /model//. Got: " + + endpoint, }, ) @@ -647,7 +666,9 @@ async def bedrock_proxy_route( aws_region_name = litellm.utils.get_secret(secret_name="AWS_REGION_NAME") if _is_bedrock_agent_runtime_route(endpoint=endpoint): # handle bedrock agents - base_target_url = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" + base_target_url = ( + f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" + ) else: return await bedrock_llm_proxy_route( endpoint=endpoint, @@ -677,7 +698,9 @@ async def bedrock_proxy_route( data = await request.json() except Exception as e: raise HTTPException(status_code=400, detail={"error": e}) - _request = AWSRequest(method="POST", url=str(updated_url), data=json.dumps(data), headers=headers) + _request = AWSRequest( + method="POST", url=str(updated_url), data=json.dumps(data), headers=headers + ) sigv4.add_auth(_request) prepped = _request.prepare() @@ -738,8 +761,14 @@ async def assemblyai_proxy_route( [Docs](https://api.assemblyai.com) """ # Set base URL based on the route - assembly_region = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url(url=str(request.url)) - base_target_url = AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region(region=assembly_region) + assembly_region = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url( + url=str(request.url) + ) + base_target_url = ( + AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region( + region=assembly_region + ) + ) encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction if not encoded_endpoint.startswith("/"): @@ -794,17 +823,79 @@ async def azure_proxy_route( Call any azure endpoint using the proxy. Just use `{PROXY_BASE_URL}/azure/{endpoint:path}` + + Checks if the deployment id in the url is a litellm model name. If so, it will route using the llm_router.allm_passthrough_route. """ + from litellm.proxy.proxy_server import llm_router + + parts = endpoint.split( + "/" + ) # azure model is in the url - e.g. https://{endpoint}/openai/deployments/{deployment-id}/completions?api-version=2024-10-21 + + if len(parts) > 1 and llm_router: + for part in parts: + is_router_model = is_passthrough_request_using_router_model( + request_body={"model": part}, llm_router=llm_router + ) + if is_router_model: + request_body = await get_request_body(request) + is_streaming_request = is_passthrough_request_streaming(request_body) + result = cast( + httpx.Response, + await llm_router.allm_passthrough_route( + model=part, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=dict(request.headers), + stream=request_body.get("stream", False), + content=None, + data=None, + files=None, + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None + ), + params=None, + headers=None, + cookies=None, + ), + ) + + if is_streaming_request: + return StreamingResponse( + content=result.aiter_bytes(), + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) + + content = await result.aread() + return Response( + content=content, + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) base_target_url = get_secret_str(secret_name="AZURE_API_BASE") if base_target_url is None: - raise Exception("Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure.") + raise Exception( + "Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure." + ) # Add or update query parameters azure_api_key = passthrough_endpoint_router.get_credentials( custom_llm_provider=litellm.LlmProviders.AZURE.value, region_name=None, ) if azure_api_key is None: - raise Exception("Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure.") + raise Exception( + "Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure." + ) return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler( endpoint=endpoint, @@ -828,7 +919,9 @@ class BaseVertexAIPassThroughHandler(ABC): @staticmethod @abstractmethod - def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str: + def update_base_target_url_with_credential_location( + base_target_url: str, vertex_location: Optional[str] + ) -> str: pass @@ -838,7 +931,9 @@ class VertexAIDiscoveryPassThroughHandler(BaseVertexAIPassThroughHandler): return "https://discoveryengine.googleapis.com/" @staticmethod - def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str: + def update_base_target_url_with_credential_location( + base_target_url: str, vertex_location: Optional[str] + ) -> str: return base_target_url @@ -848,7 +943,9 @@ class VertexAIPassThroughHandler(BaseVertexAIPassThroughHandler): return get_vertex_base_url(vertex_location) @staticmethod - def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str: + def update_base_target_url_with_credential_location( + base_target_url: str, vertex_location: Optional[str] + ) -> str: return get_vertex_base_url(vertex_location) @@ -914,14 +1011,18 @@ async def _base_vertex_proxy_route( location=vertex_location, ) - base_target_url = get_vertex_pass_through_handler.get_default_base_target_url(vertex_location) + base_target_url = get_vertex_pass_through_handler.get_default_base_target_url( + vertex_location + ) headers_passed_through = False # Use headers from the incoming request if no vertex credentials are found if vertex_credentials is None or vertex_credentials.vertex_project is None: headers = dict(request.headers) or {} headers_passed_through = True - verbose_proxy_logger.debug("default_vertex_config not set, incoming request headers %s", headers) + verbose_proxy_logger.debug( + "default_vertex_config not set, incoming request headers %s", headers + ) headers.pop("content-length", None) headers.pop("host", None) else: @@ -1087,7 +1188,9 @@ async def openai_proxy_route( region_name=None, ) if openai_api_key is None: - raise Exception("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") + raise Exception( + "Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI." + ) return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler( endpoint=endpoint, @@ -1133,7 +1236,9 @@ class BaseOpenAIPassThroughHandler: endpoint_func = create_pass_through_route( endpoint=endpoint, target=str(updated_url), - custom_headers=BaseOpenAIPassThroughHandler._assemble_headers(api_key=api_key, request=request), + custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( + api_key=api_key, request=request + ), ) # dynamically construct pass-through endpoint based on incoming path received_value = await endpoint_func( request, @@ -1150,7 +1255,10 @@ class BaseOpenAIPassThroughHandler: """ Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request """ - if RouteChecks._is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: + if ( + RouteChecks._is_assistants_api_request(request) is True + and "OpenAI-Beta" not in headers + ): headers["OpenAI-Beta"] = "assistants=v2" return headers @@ -1166,7 +1274,9 @@ class BaseOpenAIPassThroughHandler: ) @staticmethod - def _join_url_paths(base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders) -> str: + def _join_url_paths( + base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders + ) -> str: """ Properly joins a base URL with a path, preserving any existing path in the base URL. """ @@ -1182,9 +1292,14 @@ class BaseOpenAIPassThroughHandler: joined_path_str = str(base_url.copy_with(path=full_path)) # Apply OpenAI-specific path handling for both branches - if custom_llm_provider == litellm.LlmProviders.OPENAI and "/v1/" not in joined_path_str: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and "/v1/" not in joined_path_str + ): # Insert v1 after api.openai.com for OpenAI requests - joined_path_str = joined_path_str.replace("api.openai.com/", "api.openai.com/v1/") + joined_path_str = joined_path_str.replace( + "api.openai.com/", "api.openai.com/v1/" + ) return joined_path_str @@ -1231,9 +1346,7 @@ async def vertex_ai_live_websocket_passthrough( if vertex_credentials_config is not None: resolved_project = resolved_project or vertex_credentials_config.vertex_project - temp_location = ( - resolved_location or vertex_credentials_config.vertex_location - ) + temp_location = resolved_location or vertex_credentials_config.vertex_location # Ensure resolved_location is a string if isinstance(temp_location, dict): resolved_location = str(temp_location) @@ -1241,7 +1354,11 @@ async def vertex_ai_live_websocket_passthrough( resolved_location = str(temp_location) else: resolved_location = None - credentials_value = str(vertex_credentials_config.vertex_credentials) if vertex_credentials_config.vertex_credentials is not None else None + credentials_value = ( + str(vertex_credentials_config.vertex_credentials) + if vertex_credentials_config.vertex_credentials is not None + else None + ) try: resolved_location = resolved_location or ( @@ -1302,7 +1419,7 @@ async def vertex_ai_live_websocket_passthrough( # Use the new WebSocket passthrough pattern if user_api_key_dict is None: raise ValueError("user_api_key_dict is required for WebSocket passthrough") - + return await websocket_passthrough_request( websocket=websocket, target=service_url, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 23877fc9875..8aa2f407555 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3601,8 +3601,10 @@ def is_known_model(model: Optional[str], llm_router: Optional[Router]) -> bool: return False model_names = llm_router.get_model_names() + model_names_set = set(model_names) + is_in_list = False - if model in model_names: + if model in model_names_set: is_in_list = True return is_in_list From 7a61526b7ba8147581b1e61a894732ac31de99fc Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Oct 2025 13:35:00 -0700 Subject: [PATCH 2/5] feat(azure/): partially working passthrough flow via router allows azure passthrough to work for litellm models --- litellm/llms/azure/common_utils.py | 2 +- .../llms/azure/passthrough/transformation.py | 83 +++++++++++++++++++ litellm/passthrough/main.py | 10 ++- litellm/proxy/_new_secret_config.yaml | 7 +- litellm/utils.py | 55 +++++++----- 5 files changed, 132 insertions(+), 25 deletions(-) create mode 100644 litellm/llms/azure/passthrough/transformation.py diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 04448681b63..b88b6985a98 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -693,7 +693,7 @@ class BaseAzureLLM(BaseOpenAILLM): def _get_base_azure_url( api_base: Optional[str], litellm_params: Optional[Union[GenericLiteLLMParams, Dict[str, Any]]], - route: Literal["/openai/responses", "/openai/vector_stores"], + route: Union[Literal["/openai/responses", "/openai/vector_stores"], str], default_api_version: Optional[Union[str, Literal["latest", "preview"]]] = None, ) -> str: """ diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py new file mode 100644 index 00000000000..b6fed6b4470 --- /dev/null +++ b/litellm/llms/azure/passthrough/transformation.py @@ -0,0 +1,83 @@ +from typing import TYPE_CHECKING, List, Optional, Tuple + +import httpx + +from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams + +if TYPE_CHECKING: + from httpx import URL + + +class AzurePassthroughConfig(BasePassthroughConfig): + def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: + return "stream" in request_data + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + endpoint: str, + request_query_params: Optional[dict], + litellm_params: dict, + ) -> Tuple["URL", str]: + base_target_url = self.get_api_base(api_base) + + if base_target_url is None: + raise Exception("Azure api base not found") + + litellm_metadata = litellm_params.get("litellm_metadata") or {} + model_group = litellm_metadata.get("model_group") + if model_group and model_group in endpoint: + endpoint = endpoint.replace(model_group, model) + + complete_url = BaseAzureLLM._get_base_azure_url( + api_base=base_target_url, + litellm_params=litellm_params, + route=endpoint, + default_api_version=litellm_params.get("api_version"), + ) + return ( + httpx.URL(complete_url), + base_target_url, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + return BaseAzureLLM._base_validate_azure_environment( + headers=headers, + litellm_params=GenericLiteLLMParams(**litellm_params), + ) + + @staticmethod + def get_api_base( + api_base: Optional[str] = None, + ) -> Optional[str]: + return api_base or get_secret_str("AZURE_API_BASE") + + @staticmethod + def get_api_key( + api_key: Optional[str] = None, + ) -> Optional[str]: + return api_key or get_secret_str("AZURE_API_KEY") + + @staticmethod + def get_base_model(model: str) -> Optional[str]: + return model + + def get_models( + self, api_key: Optional[str] = None, api_base: Optional[str] = None + ) -> List[str]: + return super().get_models(api_key, api_base) diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index f4dc1ef6c84..b4a76822022 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -242,12 +242,14 @@ def llm_passthrough_route( request_query_params=request_query_params, litellm_params=litellm_params_dict, ) - - # need to encode the id of application-inference-profile for bedrock + + # [TODO: Refactor to bedrockpassthroughconfig] need to encode the id of application-inference-profile for bedrock if custom_llm_provider == "bedrock" and "application-inference-profile" in endpoint: - encoded_url_str = CommonUtils.encode_bedrock_runtime_modelid_arn(str(updated_url)) + encoded_url_str = CommonUtils.encode_bedrock_runtime_modelid_arn( + str(updated_url) + ) updated_url = httpx.URL(encoded_url_str) - + # Add or update query parameters provider_api_key = provider_config.get_api_key(api_key) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 8c929fac8d4..0882a9298aa 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -15,7 +15,12 @@ model_list: model: hosted_vllm/whisper-v3 api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" api_key: dummy - + - model_name: azure-hidden-model + litellm_params: + model: azure/gpt-4.1 + api_base: os.environ/AZURE_API_BASE_ALT + api_version: "2023-05-15" + api_key: os.environ/AZURE_API_KEY_HIDDEN # mcp_servers: # github_mcp: diff --git a/litellm/utils.py b/litellm/utils.py index 3c6c3ac86e4..7c712b90284 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -532,9 +532,6 @@ def get_dynamic_callbacks( return returned_callbacks - - - def function_setup( # noqa: PLR0915 original_function: str, rules_obj, start_time, *args, **kwargs ): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc. @@ -802,7 +799,7 @@ def function_setup( # noqa: PLR0915 call_type=call_type, ): stream = True - logging_obj = get_litellm_logging_class()( # Victim for object pool + logging_obj = get_litellm_logging_class()( # Victim for object pool model=model, # type: ignore messages=messages, stream=stream, @@ -1417,7 +1414,8 @@ def client(original_function): # noqa: PLR0915 if _caching_handler_response is not None: if ( _caching_handler_response.cached_result is not None - and _caching_handler_response.final_embedding_cached_response is None + and _caching_handler_response.final_embedding_cached_response + is None ): return _caching_handler_response.cached_result @@ -1683,7 +1681,6 @@ def _is_streaming_request( return False - def _select_tokenizer( model: str, custom_tokenizer: Optional[CustomHuggingfaceTokenizer] = None ): @@ -4867,16 +4864,24 @@ def _get_model_info_helper( # noqa: PLR0915 max_input_tokens=_model_info.get("max_input_tokens", None), max_output_tokens=_model_info.get("max_output_tokens", None), input_cost_per_token=_input_cost_per_token, - input_cost_per_token_flex=_model_info.get("input_cost_per_token_flex", None), - input_cost_per_token_priority=_model_info.get("input_cost_per_token_priority", None), + input_cost_per_token_flex=_model_info.get( + "input_cost_per_token_flex", None + ), + input_cost_per_token_priority=_model_info.get( + "input_cost_per_token_priority", None + ), cache_creation_input_token_cost=_model_info.get( "cache_creation_input_token_cost", None ), cache_read_input_token_cost=_model_info.get( "cache_read_input_token_cost", None ), - cache_read_input_token_cost_flex=_model_info.get("cache_read_input_token_cost_flex", None), - cache_read_input_token_cost_priority=_model_info.get("cache_read_input_token_cost_priority", None), + cache_read_input_token_cost_flex=_model_info.get( + "cache_read_input_token_cost_flex", None + ), + cache_read_input_token_cost_priority=_model_info.get( + "cache_read_input_token_cost_priority", None + ), cache_creation_input_token_cost_above_1hr=_model_info.get( "cache_creation_input_token_cost_above_1hr", None ), @@ -4901,8 +4906,12 @@ def _get_model_info_helper( # noqa: PLR0915 "output_cost_per_token_batches" ), output_cost_per_token=_output_cost_per_token, - output_cost_per_token_flex=_model_info.get("output_cost_per_token_flex", None), - output_cost_per_token_priority=_model_info.get("output_cost_per_token_priority", None), + output_cost_per_token_flex=_model_info.get( + "output_cost_per_token_flex", None + ), + output_cost_per_token_priority=_model_info.get( + "output_cost_per_token_priority", None + ), output_cost_per_audio_token=_model_info.get( "output_cost_per_audio_token", None ), @@ -6434,7 +6443,7 @@ def get_valid_models( try: ################################ - # init litellm_params + # init litellm_params ################################# if litellm_params is None: litellm_params = LiteLLM_Params(model="") @@ -6443,7 +6452,7 @@ def get_valid_models( if api_base is not None: litellm_params.api_base = api_base ################################# - + check_provider_endpoint = ( check_provider_endpoint or litellm.check_provider_endpoint ) @@ -6918,7 +6927,10 @@ class ProviderConfigManager: return litellm.LlamaAPIConfig() elif litellm.LlmProviders.TEXT_COMPLETION_OPENAI == provider: return litellm.OpenAITextCompletionConfig() - elif litellm.LlmProviders.COHERE_CHAT == provider or litellm.LlmProviders.COHERE == provider: + elif ( + litellm.LlmProviders.COHERE_CHAT == provider + or litellm.LlmProviders.COHERE == provider + ): return litellm.CohereChatConfig() elif litellm.LlmProviders.SNOWFLAKE == provider: return litellm.SnowflakeConfig() @@ -7345,7 +7357,12 @@ class ProviderConfigManager: ) return VLLMPassthroughConfig() + elif LlmProviders.AZURE == provider: + from litellm.llms.azure.passthrough.transformation import ( + AzurePassthroughConfig, + ) + return AzurePassthroughConfig() return None @staticmethod @@ -7532,9 +7549,7 @@ class ProviderConfigManager: return RecraftImageEditConfig() elif LlmProviders.AZURE_AI == provider: - from litellm.llms.azure_ai.image_edit import ( - get_azure_ai_image_edit_config, - ) + from litellm.llms.azure_ai.image_edit import get_azure_ai_image_edit_config return get_azure_ai_image_edit_config(model) elif LlmProviders.LITELLM_PROXY == provider: @@ -7589,7 +7604,9 @@ def get_end_user_id_for_cost_tracking( service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking. """ - _metadata = cast(dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params))) + _metadata = cast( + dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params)) + ) end_user_id = cast( Optional[str], From 6177b23ebfb2325174806605c6b47181ddae84f7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Oct 2025 13:44:10 -0700 Subject: [PATCH 3/5] feat(azure/passthrough): working azure passthrough with router models for openai calls Fixes issue where azure passthrough could not work with workload identity + litellm model name --- litellm/llms/azure/common_utils.py | 4 ---- litellm/llms/azure/passthrough/transformation.py | 4 +++- litellm/proxy/_new_secret_config.yaml | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index b88b6985a98..3f1785d885c 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -665,10 +665,6 @@ class BaseAzureLLM(BaseOpenAILLM): ) -> dict: litellm_params = litellm_params or GenericLiteLLMParams() - # If api-key is already in headers, preserve it - if "api-key" in headers: - return headers - api_key = ( litellm_params.api_key or litellm.api_key diff --git a/litellm/llms/azure/passthrough/transformation.py b/litellm/llms/azure/passthrough/transformation.py index b6fed6b4470..4e9de4b314f 100644 --- a/litellm/llms/azure/passthrough/transformation.py +++ b/litellm/llms/azure/passthrough/transformation.py @@ -58,7 +58,9 @@ class AzurePassthroughConfig(BasePassthroughConfig): ) -> dict: return BaseAzureLLM._base_validate_azure_environment( headers=headers, - litellm_params=GenericLiteLLMParams(**litellm_params), + litellm_params=GenericLiteLLMParams( + **{**litellm_params, "api_key": api_key} + ), ) @staticmethod diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 0882a9298aa..fad6ad336a8 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -20,7 +20,7 @@ model_list: model: azure/gpt-4.1 api_base: os.environ/AZURE_API_BASE_ALT api_version: "2023-05-15" - api_key: os.environ/AZURE_API_KEY_HIDDEN + api_key: os.environ/AZURE_API_KEY_ALT # mcp_servers: # github_mcp: From d7aab1d5dec0ddd1158d5fd32dc79eb14fbca359 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Oct 2025 13:53:36 -0700 Subject: [PATCH 4/5] test: add unit tests --- .../passthrough/test_passthrough_main.py | 228 +++++++++++++----- 1 file changed, 164 insertions(+), 64 deletions(-) diff --git a/tests/test_litellm/passthrough/test_passthrough_main.py b/tests/test_litellm/passthrough/test_passthrough_main.py index 27defe0eb0b..2f577b17a05 100644 --- a/tests/test_litellm/passthrough/test_passthrough_main.py +++ b/tests/test_litellm/passthrough/test_passthrough_main.py @@ -53,27 +53,38 @@ def test_llm_passthrough_route(): def test_bedrock_application_inference_profile_url_encoding(): client = HTTPHandler() - + mock_provider_config = MagicMock() mock_provider_config.get_complete_url.return_value = ( - httpx.URL("https://bedrock-runtime.us-east-1.amazonaws.com/model/arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd/converse"), - "https://bedrock-runtime.us-east-1.amazonaws.com" + httpx.URL( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd/converse" + ), + "https://bedrock-runtime.us-east-1.amazonaws.com", ) mock_provider_config.get_api_key.return_value = "test-key" mock_provider_config.validate_environment.return_value = {} mock_provider_config.sign_request.return_value = ({}, None) mock_provider_config.is_streaming_request.return_value = False - with patch("litellm.utils.ProviderConfigManager.get_provider_passthrough_config", return_value=mock_provider_config), \ - patch("litellm.litellm_core_utils.get_litellm_params.get_litellm_params", return_value={}), \ - patch("litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", return_value=("test-model", "bedrock", "test-key", "test-base")), \ - patch.object(client.client, "send", return_value=MagicMock(status_code=200)) as mock_send, \ - patch.object(client.client, "build_request") as mock_build_request: - + with patch( + "litellm.utils.ProviderConfigManager.get_provider_passthrough_config", + return_value=mock_provider_config, + ), patch( + "litellm.litellm_core_utils.get_litellm_params.get_litellm_params", + return_value={}, + ), patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("test-model", "bedrock", "test-key", "test-base"), + ), patch.object( + client.client, "send", return_value=MagicMock(status_code=200) + ) as mock_send, patch.object( + client.client, "build_request" + ) as mock_build_request: + # Mock logging object mock_logging_obj = MagicMock() mock_logging_obj.update_environment_variables = MagicMock() - + response = llm_passthrough_route( model="arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd", endpoint="model/arn:aws:bedrock:us-east-1:123456789123:application-inference-profile/r742sbn2zckd/converse", @@ -86,7 +97,7 @@ def test_bedrock_application_inference_profile_url_encoding(): # Verify that build_request was called with the encoded URL mock_build_request.assert_called_once() call_args = mock_build_request.call_args - + # The URL should have the application-inference-profile ID encoded actual_url = str(call_args.kwargs["url"]) assert "application-inference-profile%2Fr742sbn2zckd" in actual_url @@ -95,28 +106,39 @@ def test_bedrock_application_inference_profile_url_encoding(): def test_bedrock_non_application_inference_profile_no_encoding(): client = HTTPHandler() - + # Mock the provider config and its methods mock_provider_config = MagicMock() mock_provider_config.get_complete_url.return_value = ( - httpx.URL("https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet-20240229-v1:0/converse"), - "https://bedrock-runtime.us-east-1.amazonaws.com" + httpx.URL( + "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-3-sonnet-20240229-v1:0/converse" + ), + "https://bedrock-runtime.us-east-1.amazonaws.com", ) mock_provider_config.get_api_key.return_value = "test-key" mock_provider_config.validate_environment.return_value = {} mock_provider_config.sign_request.return_value = ({}, None) mock_provider_config.is_streaming_request.return_value = False - with patch("litellm.utils.ProviderConfigManager.get_provider_passthrough_config", return_value=mock_provider_config), \ - patch("litellm.litellm_core_utils.get_litellm_params.get_litellm_params", return_value={}), \ - patch("litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", return_value=("test-model", "bedrock", "test-key", "test-base")), \ - patch.object(client.client, "send", return_value=MagicMock(status_code=200)) as mock_send, \ - patch.object(client.client, "build_request") as mock_build_request: - + with patch( + "litellm.utils.ProviderConfigManager.get_provider_passthrough_config", + return_value=mock_provider_config, + ), patch( + "litellm.litellm_core_utils.get_litellm_params.get_litellm_params", + return_value={}, + ), patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("test-model", "bedrock", "test-key", "test-base"), + ), patch.object( + client.client, "send", return_value=MagicMock(status_code=200) + ) as mock_send, patch.object( + client.client, "build_request" + ) as mock_build_request: + # Mock logging object mock_logging_obj = MagicMock() mock_logging_obj.update_environment_variables = MagicMock() - + response = llm_passthrough_route( model="anthropic.claude-3-sonnet-20240229-v1:0", endpoint="model/anthropic.claude-3-sonnet-20240229-v1:0/converse", @@ -129,7 +151,7 @@ def test_bedrock_non_application_inference_profile_no_encoding(): # Verify that build_request was called with the original URL (no encoding) mock_build_request.assert_called_once() call_args = mock_build_request.call_args - + # The URL should NOT have application-inference-profile encoding actual_url = str(call_args.kwargs["url"]) assert "application-inference-profile%2F" not in actual_url @@ -151,21 +173,21 @@ def test_update_stream_param_based_on_request_body(): parsed_body=parsed_body, stream=False ) assert result is True - + # Test 2: no stream in request body should return original stream param parsed_body = {"model": "test-model"} result = HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body( parsed_body=parsed_body, stream=False ) assert result is False - + # Test 3: stream=False in request body should return False parsed_body = {"stream": False, "model": "test-model"} result = HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body( parsed_body=parsed_body, stream=True ) assert result is False - + # Test 4: no stream param provided, no stream in body parsed_body = {"model": "test-model"} result = HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body( @@ -178,14 +200,14 @@ def test_update_stream_param_based_on_request_body(): def mock_request(): """Create a mock request with headers""" from typing import Optional - + class QueryParams: def __init__(self): self._dict = {} - + def __iter__(self): return iter(self._dict) - + def items(self): return self._dict.items() @@ -210,6 +232,7 @@ def mock_request(): def mock_user_api_key_dict(): """Create a mock user API key dictionary""" from litellm.proxy._types import UserAPIKeyAuth + return UserAPIKeyAuth( api_key="test-key", user_id="test-user", @@ -223,8 +246,8 @@ async def test_pass_through_request_stream_param_override( mock_request, mock_user_api_key_dict ): """ - Test that when stream=None is passed as parameter but stream=True - is in request body, the request body value takes precedence and + Test that when stream=None is passed as parameter but stream=True + is in request body, the request body value takes precedence and the eventual POST request uses streaming. """ from unittest.mock import AsyncMock, Mock, patch @@ -238,29 +261,29 @@ async def test_pass_through_request_stream_param_override( "model": "claude-3-5-sonnet-20241022", "max_tokens": 256, "messages": [{"role": "user", "content": "Hello, world"}], - "stream": True # This should override the function parameter + "stream": True, # This should override the function parameter } # Create a mock streaming response mock_response = AsyncMock() mock_response.status_code = 200 mock_response.headers = {"content-type": "text/event-stream"} - + # Mock the streaming response behavior async def mock_aiter_bytes(): yield b'data: {"content": "Hello"}\n\n' yield b'data: {"content": "World"}\n\n' - yield b'data: [DONE]\n\n' - + yield b"data: [DONE]\n\n" + mock_response.aiter_bytes = mock_aiter_bytes # Create mocks for the async client mock_async_client = AsyncMock() mock_request_obj = AsyncMock() - + # Mock build_request to return a request object (it's a sync method) mock_async_client.build_request = Mock(return_value=mock_request_obj) - + # Mock send to return the streaming response mock_async_client.send.return_value = mock_response @@ -269,9 +292,7 @@ async def test_pass_through_request_stream_param_override( mock_client_obj.client = mock_async_client # Create the request - request = mock_request( - headers={}, method="POST", request_body=request_body - ) + request = mock_request(headers={}, method="POST", request_body=request_body) with patch( "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client", @@ -298,33 +319,32 @@ async def test_pass_through_request_stream_param_override( httpx.URL("https://api.anthropic.com/v1/messages"), json=request_body, params={}, - headers={ - "Authorization": "Bearer test-key" - }, + headers={"Authorization": "Bearer test-key"}, ) - + # Verify that send was called with stream=True mock_async_client.send.assert_called_once_with( - mock_request_obj, - stream=True # This proves that stream=True from request body was used + mock_request_obj, + stream=True, # This proves that stream=True from request body was used ) - + # Verify that the non-streaming request method was NOT called mock_async_client.request.assert_not_called() - + # Verify response is a StreamingResponse from fastapi.responses import StreamingResponse + assert isinstance(response, StreamingResponse) assert response.status_code == 200 -@pytest.mark.asyncio +@pytest.mark.asyncio async def test_pass_through_request_stream_param_no_override( mock_request, mock_user_api_key_dict ): """ - Test that when stream=False is passed as parameter and no stream - is in request body, the function parameter is used and + Test that when stream=False is passed as parameter and no stream + is in request body, the function parameter is used and the eventual request uses non-streaming. """ from unittest.mock import AsyncMock, Mock, patch @@ -335,7 +355,7 @@ async def test_pass_through_request_stream_param_no_override( # Create request body without stream parameter request_body = { - "model": "claude-3-5-sonnet-20241022", + "model": "claude-3-5-sonnet-20241022", "max_tokens": 256, "messages": [{"role": "user", "content": "Hello, world"}], # No stream parameter - should use function parameter stream=False @@ -346,15 +366,15 @@ async def test_pass_through_request_stream_param_no_override( mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} mock_response._content = b'{"response": "Hello world"}' - + async def mock_aread(): return mock_response._content - + mock_response.aread = mock_aread # Create mocks for the async client mock_async_client = AsyncMock() - + # Mock request to return the non-streaming response mock_async_client.request.return_value = mock_response @@ -363,9 +383,7 @@ async def test_pass_through_request_stream_param_no_override( mock_client_obj.client = mock_async_client # Create the request - request = mock_request( - headers={}, method="POST", request_body=request_body - ) + request = mock_request(headers={}, method="POST", request_body=request_body) with patch( "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client", @@ -388,23 +406,105 @@ async def test_pass_through_request_stream_param_no_override( # Verify that build_request was NOT called (no streaming path) mock_async_client.build_request.assert_not_called() - + # Verify that send was NOT called (no streaming path) mock_async_client.send.assert_not_called() - + # Verify that the non-streaming request method WAS called mock_async_client.request.assert_called_once_with( method="POST", url=httpx.URL("https://api.anthropic.com/v1/messages"), - headers={ - "Authorization": "Bearer test-key" - }, + headers={"Authorization": "Bearer test-key"}, params={}, json=request_body, ) - + # Verify response is a regular Response (not StreamingResponse) from fastapi.responses import Response, StreamingResponse + assert not isinstance(response, StreamingResponse) assert isinstance(response, Response) - assert response.status_code == 200 \ No newline at end of file + assert response.status_code == 200 + + +def test_azure_with_custom_api_base_and_key(): + """ + Test that llm_passthrough_route correctly handles Azure OpenAI + with custom api_base and api_key. + """ + client = HTTPHandler() + + # Mock the provider config and its methods + mock_provider_config = MagicMock() + mock_provider_config.get_complete_url.return_value = ( + httpx.URL( + "https://my-custom-base/openai/deployments/gpt-4.1/chat/completions?api-version=2024-02-01" + ), + "https://my-custom-base", + ) + mock_provider_config.get_api_key.return_value = "my-custom-key" + mock_provider_config.validate_environment.return_value = { + "api-key": "my-custom-key" + } + mock_provider_config.sign_request.return_value = ( + {"api-key": "my-custom-key"}, + None, + ) + mock_provider_config.is_streaming_request.return_value = False + + with patch( + "litellm.utils.ProviderConfigManager.get_provider_passthrough_config", + return_value=mock_provider_config, + ), patch( + "litellm.litellm_core_utils.get_litellm_params.get_litellm_params", + return_value={}, + ), patch( + "litellm.litellm_core_utils.get_llm_provider_logic.get_llm_provider", + return_value=("gpt-4.1", "azure", "my-custom-key", "https://my-custom-base"), + ), patch.object( + client.client, + "send", + return_value=MagicMock( + status_code=200, json=lambda: {"id": "chatcmpl-123", "choices": []} + ), + ) as mock_send, patch.object( + client.client, "build_request" + ) as mock_build_request: + + # Mock logging object + mock_logging_obj = MagicMock() + mock_logging_obj.update_environment_variables = MagicMock() + + response = llm_passthrough_route( + model="azure/gpt-4.1", + endpoint="openai/deployments/gpt-4.1/chat/completions", + method="POST", + custom_llm_provider="azure", + api_base="https://my-custom-base", + api_key="my-custom-key", + json={ + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "Hello!"}], + }, + client=client, + litellm_logging_obj=mock_logging_obj, + ) + + # Verify that build_request was called with the correct parameters + mock_build_request.assert_called_once() + call_args = mock_build_request.call_args + + # Verify the URL contains the custom base + actual_url = str(call_args.kwargs["url"]) + assert "my-custom-base" in actual_url + assert "gpt-4.1" in actual_url + + # Verify the headers contain the custom API key + headers = call_args.kwargs["headers"] + assert headers["api-key"] == "my-custom-key" + + # Verify the model in JSON body is updated + json_body = call_args.kwargs["json"] + assert json_body["model"] == "gpt-4.1" + + assert response.status_code == 200 From 6f671d2bea10ddd41ae24956b13ee8b540b6a9ba Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 6 Oct 2025 13:56:30 -0700 Subject: [PATCH 5/5] feat(llm_passthrough_endpoints.py): working streaming on azure passthrough via router --- .../llm_passthrough_endpoints.py | 68 +++++++++++-------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 849eac17f07..d07bfbb11ae 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -840,39 +840,51 @@ async def azure_proxy_route( if is_router_model: request_body = await get_request_body(request) is_streaming_request = is_passthrough_request_streaming(request_body) - result = cast( - httpx.Response, - await llm_router.allm_passthrough_route( - model=part, - method=request.method, - endpoint=endpoint, - request_query_params=request.query_params, - request_headers=dict(request.headers), - stream=request_body.get("stream", False), - content=None, - data=None, - files=None, - json=( - request_body - if request.headers.get("content-type") == "application/json" - else None - ), - params=None, - headers=None, - cookies=None, + result = await llm_router.allm_passthrough_route( + model=part, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=dict(request.headers), + stream=request_body.get("stream", False), + content=None, + data=None, + files=None, + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None ), + params=None, + headers=None, + cookies=None, ) if is_streaming_request: - return StreamingResponse( - content=result.aiter_bytes(), - status_code=result.status_code, - headers=HttpPassThroughEndpointHelpers.get_response_headers( - headers=result.headers, - custom_headers=None, - ), - ) + # Check if result is an async generator (from _async_streaming) + import inspect + if inspect.isasyncgen(result): + # Result is already an async generator, use it directly + return StreamingResponse( + content=result, + status_code=200, + headers={"content-type": "text/event-stream"}, + ) + else: + # Result is an httpx.Response, use aiter_bytes() + result = cast(httpx.Response, result) + return StreamingResponse( + content=result.aiter_bytes(), + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) + + # Non-streaming response + result = cast(httpx.Response, result) content = await result.aread() return Response( content=content,