diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7ed04dc79c9..fd09aca4c1a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -22173,6 +22173,307 @@ "supports_tool_choice": true, "supports_vision": false }, + "watsonx/bigscience/mt0-xxl-13b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/core42/jais-13b-chat": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/google/flan-t5-xl-3b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.00025, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-13b-chat-v2": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-13b-instruct-v2": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-3-3-8b-instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/ibm/granite-4-h-small": { + "max_tokens": 20480, + "max_input_tokens": 20480, + "max_output_tokens": 20480, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.0025, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/ibm/granite-guardian-3-2-2b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-guardian-3-3-8b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-ttm-1024-96-r2": { + "max_tokens": 512, + "max_input_tokens": 512, + "max_output_tokens": 512, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.000625, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-ttm-1536-96-r2": { + "max_tokens": 512, + "max_input_tokens": 512, + "max_output_tokens": 512, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.000625, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-ttm-512-96-r2": { + "max_tokens": 512, + "max_input_tokens": 512, + "max_output_tokens": 512, + "input_cost_per_token": 0.000625, + "output_cost_per_token": 0.000625, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/ibm/granite-vision-3-2-2b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": true + }, + "watsonx/meta-llama/llama-3-2-11b-vision-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "watsonx/meta-llama/llama-3-2-1b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0001, + "output_cost_per_token": 0.0002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-3-2-3b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-3-2-90b-vision-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.008, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true + }, + "watsonx/meta-llama/llama-3-3-70b-instruct": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.002, + "output_cost_per_token": 0.006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-4-maverick-17b": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/meta-llama/llama-guard-3-11b-vision": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00025, + "output_cost_per_token": 0.001, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": true + }, + "watsonx/mistralai/mistral-medium-2505": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00225, + "output_cost_per_token": 0.00675, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/mistralai/mistral-small-2503": { + "max_tokens": 32000, + "max_input_tokens": 32000, + "max_output_tokens": 32000, + "input_cost_per_token": 0.0002, + "output_cost_per_token": 0.0006, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": false + }, + "watsonx/mistralai/pixtral-12b-2409": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.00015, + "output_cost_per_token": 0.00015, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": true + }, + "watsonx/openai/gpt-oss-120b": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.004, + "output_cost_per_token": 0.016, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "watsonx/sdaia/allam-1-13b-instruct": { + "max_tokens": 8192, + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0005, + "output_cost_per_token": 0.002, + "litellm_provider": "watsonx", + "mode": "chat", + "supports_function_calling": false, + "supports_parallel_function_calling": false, + "supports_vision": false + }, + "whisper-1": { "input_cost_per_second": 0.0001, "litellm_provider": "openai", diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index b7b30d36f99..8c929fac8d4 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -16,6 +16,7 @@ model_list: api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5" api_key: dummy + # mcp_servers: # github_mcp: # url: "https://api.githubcopilot.com/mcp" diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 8aa3b90d954..849eac17f07 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -21,9 +21,7 @@ from litellm.constants import BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.proxy._types import * from litellm.proxy.auth.route_checks import RouteChecks -from litellm.proxy.auth.user_api_key_auth import ( - user_api_key_auth, -) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.http_parsing_utils import ( _read_request_body, get_form_data, @@ -31,6 +29,7 @@ from litellm.proxy.common_utils.http_parsing_utils import ( ) from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + HttpPassThroughEndpointHelpers, create_pass_through_route, create_websocket_passthrough_route, websocket_passthrough_request, @@ -57,7 +56,9 @@ def create_request_copy(request: Request): } -def is_passthrough_request_using_router_model(request_body: dict, llm_router: Optional[litellm.Router]) -> bool: +def is_passthrough_request_using_router_model( + request_body: dict, llm_router: Optional[litellm.Router] +) -> bool: """ Returns True if the model is in the llm_router model names """ @@ -93,12 +94,16 @@ async def llm_passthrough_factory_proxy_route( model=None, ) if provider_config is None: - raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} not found") + raise HTTPException( + status_code=404, detail=f"Provider {custom_llm_provider} not found" + ) base_target_url = provider_config.get_api_base() if base_target_url is None: - raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} api base not found") + raise HTTPException( + status_code=404, detail=f"Provider {custom_llm_provider} api base not found" + ) encoded_endpoint = httpx.URL(endpoint).path @@ -177,11 +182,17 @@ async def gemini_proxy_route( [Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio) """ ## CHECK FOR LITELLM API KEY IN THE QUERY PARAMS - ?..key=LITELLM_API_KEY - google_ai_studio_api_key = request.query_params.get("key") or request.headers.get("x-goog-api-key") + google_ai_studio_api_key = request.query_params.get("key") or request.headers.get( + "x-goog-api-key" + ) - user_api_key_dict = await user_api_key_auth(request=request, api_key=f"Bearer {google_ai_studio_api_key}") + user_api_key_dict = await user_api_key_auth( + request=request, api_key=f"Bearer {google_ai_studio_api_key}" + ) - base_target_url = os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com" + base_target_url = ( + os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com" + ) encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction @@ -293,13 +304,12 @@ async def vllm_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/vllm) """ - from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( - HttpPassThroughEndpointHelpers, - ) from litellm.proxy.proxy_server import llm_router request_body = await get_request_body(request) - is_router_model = is_passthrough_request_using_router_model(request_body, llm_router) + is_router_model = is_passthrough_request_using_router_model( + request_body, llm_router + ) is_streaming_request = is_passthrough_request_streaming(request_body) if is_router_model and llm_router: result = cast( @@ -314,7 +324,11 @@ async def vllm_proxy_route( content=None, data=None, files=None, - json=(request_body if request.headers.get("content-type") == "application/json" else None), + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None + ), params=None, headers=None, cookies=None, @@ -492,7 +506,9 @@ async def handle_bedrock_count_tokens( # Extract model from request body model = request_body.get("model") if not model: - raise HTTPException(status_code=400, detail={"error": "Model is required in request body"}) + raise HTTPException( + status_code=400, detail={"error": "Model is required in request body"} + ) # Get model parameters from router litellm_params = {"user_api_key_dict": user_api_key_dict} @@ -531,7 +547,9 @@ async def handle_bedrock_count_tokens( raise except Exception as e: verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {str(e)}") - raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {str(e)}"}) + raise HTTPException( + status_code=500, detail={"error": f"CountTokens processing error: {str(e)}"} + ) async def bedrock_llm_proxy_route( @@ -583,7 +601,8 @@ async def bedrock_llm_proxy_route( raise HTTPException( status_code=400, detail={ - "error": "Model missing from endpoint. Expected format: /model//. Got: " + endpoint, + "error": "Model missing from endpoint. Expected format: /model//. Got: " + + endpoint, }, ) @@ -647,7 +666,9 @@ async def bedrock_proxy_route( aws_region_name = litellm.utils.get_secret(secret_name="AWS_REGION_NAME") if _is_bedrock_agent_runtime_route(endpoint=endpoint): # handle bedrock agents - base_target_url = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" + base_target_url = ( + f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com" + ) else: return await bedrock_llm_proxy_route( endpoint=endpoint, @@ -677,7 +698,9 @@ async def bedrock_proxy_route( data = await request.json() except Exception as e: raise HTTPException(status_code=400, detail={"error": e}) - _request = AWSRequest(method="POST", url=str(updated_url), data=json.dumps(data), headers=headers) + _request = AWSRequest( + method="POST", url=str(updated_url), data=json.dumps(data), headers=headers + ) sigv4.add_auth(_request) prepped = _request.prepare() @@ -738,8 +761,14 @@ async def assemblyai_proxy_route( [Docs](https://api.assemblyai.com) """ # Set base URL based on the route - assembly_region = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url(url=str(request.url)) - base_target_url = AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region(region=assembly_region) + assembly_region = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url( + url=str(request.url) + ) + base_target_url = ( + AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region( + region=assembly_region + ) + ) encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction if not encoded_endpoint.startswith("/"): @@ -794,17 +823,79 @@ async def azure_proxy_route( Call any azure endpoint using the proxy. Just use `{PROXY_BASE_URL}/azure/{endpoint:path}` + + Checks if the deployment id in the url is a litellm model name. If so, it will route using the llm_router.allm_passthrough_route. """ + from litellm.proxy.proxy_server import llm_router + + parts = endpoint.split( + "/" + ) # azure model is in the url - e.g. https://{endpoint}/openai/deployments/{deployment-id}/completions?api-version=2024-10-21 + + if len(parts) > 1 and llm_router: + for part in parts: + is_router_model = is_passthrough_request_using_router_model( + request_body={"model": part}, llm_router=llm_router + ) + if is_router_model: + request_body = await get_request_body(request) + is_streaming_request = is_passthrough_request_streaming(request_body) + result = cast( + httpx.Response, + await llm_router.allm_passthrough_route( + model=part, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=dict(request.headers), + stream=request_body.get("stream", False), + content=None, + data=None, + files=None, + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None + ), + params=None, + headers=None, + cookies=None, + ), + ) + + if is_streaming_request: + return StreamingResponse( + content=result.aiter_bytes(), + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) + + content = await result.aread() + return Response( + content=content, + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) base_target_url = get_secret_str(secret_name="AZURE_API_BASE") if base_target_url is None: - raise Exception("Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure.") + raise Exception( + "Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure." + ) # Add or update query parameters azure_api_key = passthrough_endpoint_router.get_credentials( custom_llm_provider=litellm.LlmProviders.AZURE.value, region_name=None, ) if azure_api_key is None: - raise Exception("Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure.") + raise Exception( + "Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure." + ) return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler( endpoint=endpoint, @@ -828,7 +919,9 @@ class BaseVertexAIPassThroughHandler(ABC): @staticmethod @abstractmethod - def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str: + def update_base_target_url_with_credential_location( + base_target_url: str, vertex_location: Optional[str] + ) -> str: pass @@ -838,7 +931,9 @@ class VertexAIDiscoveryPassThroughHandler(BaseVertexAIPassThroughHandler): return "https://discoveryengine.googleapis.com/" @staticmethod - def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str: + def update_base_target_url_with_credential_location( + base_target_url: str, vertex_location: Optional[str] + ) -> str: return base_target_url @@ -848,7 +943,9 @@ class VertexAIPassThroughHandler(BaseVertexAIPassThroughHandler): return get_vertex_base_url(vertex_location) @staticmethod - def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str: + def update_base_target_url_with_credential_location( + base_target_url: str, vertex_location: Optional[str] + ) -> str: return get_vertex_base_url(vertex_location) @@ -914,14 +1011,18 @@ async def _base_vertex_proxy_route( location=vertex_location, ) - base_target_url = get_vertex_pass_through_handler.get_default_base_target_url(vertex_location) + base_target_url = get_vertex_pass_through_handler.get_default_base_target_url( + vertex_location + ) headers_passed_through = False # Use headers from the incoming request if no vertex credentials are found if vertex_credentials is None or vertex_credentials.vertex_project is None: headers = dict(request.headers) or {} headers_passed_through = True - verbose_proxy_logger.debug("default_vertex_config not set, incoming request headers %s", headers) + verbose_proxy_logger.debug( + "default_vertex_config not set, incoming request headers %s", headers + ) headers.pop("content-length", None) headers.pop("host", None) else: @@ -1087,7 +1188,9 @@ async def openai_proxy_route( region_name=None, ) if openai_api_key is None: - raise Exception("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.") + raise Exception( + "Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI." + ) return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler( endpoint=endpoint, @@ -1133,7 +1236,9 @@ class BaseOpenAIPassThroughHandler: endpoint_func = create_pass_through_route( endpoint=endpoint, target=str(updated_url), - custom_headers=BaseOpenAIPassThroughHandler._assemble_headers(api_key=api_key, request=request), + custom_headers=BaseOpenAIPassThroughHandler._assemble_headers( + api_key=api_key, request=request + ), ) # dynamically construct pass-through endpoint based on incoming path received_value = await endpoint_func( request, @@ -1150,7 +1255,10 @@ class BaseOpenAIPassThroughHandler: """ Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request """ - if RouteChecks._is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers: + if ( + RouteChecks._is_assistants_api_request(request) is True + and "OpenAI-Beta" not in headers + ): headers["OpenAI-Beta"] = "assistants=v2" return headers @@ -1166,7 +1274,9 @@ class BaseOpenAIPassThroughHandler: ) @staticmethod - def _join_url_paths(base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders) -> str: + def _join_url_paths( + base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders + ) -> str: """ Properly joins a base URL with a path, preserving any existing path in the base URL. """ @@ -1182,9 +1292,14 @@ class BaseOpenAIPassThroughHandler: joined_path_str = str(base_url.copy_with(path=full_path)) # Apply OpenAI-specific path handling for both branches - if custom_llm_provider == litellm.LlmProviders.OPENAI and "/v1/" not in joined_path_str: + if ( + custom_llm_provider == litellm.LlmProviders.OPENAI + and "/v1/" not in joined_path_str + ): # Insert v1 after api.openai.com for OpenAI requests - joined_path_str = joined_path_str.replace("api.openai.com/", "api.openai.com/v1/") + joined_path_str = joined_path_str.replace( + "api.openai.com/", "api.openai.com/v1/" + ) return joined_path_str @@ -1231,9 +1346,7 @@ async def vertex_ai_live_websocket_passthrough( if vertex_credentials_config is not None: resolved_project = resolved_project or vertex_credentials_config.vertex_project - temp_location = ( - resolved_location or vertex_credentials_config.vertex_location - ) + temp_location = resolved_location or vertex_credentials_config.vertex_location # Ensure resolved_location is a string if isinstance(temp_location, dict): resolved_location = str(temp_location) @@ -1241,7 +1354,11 @@ async def vertex_ai_live_websocket_passthrough( resolved_location = str(temp_location) else: resolved_location = None - credentials_value = str(vertex_credentials_config.vertex_credentials) if vertex_credentials_config.vertex_credentials is not None else None + credentials_value = ( + str(vertex_credentials_config.vertex_credentials) + if vertex_credentials_config.vertex_credentials is not None + else None + ) try: resolved_location = resolved_location or ( @@ -1302,7 +1419,7 @@ async def vertex_ai_live_websocket_passthrough( # Use the new WebSocket passthrough pattern if user_api_key_dict is None: raise ValueError("user_api_key_dict is required for WebSocket passthrough") - + return await websocket_passthrough_request( websocket=websocket, target=service_url, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 23877fc9875..8aa2f407555 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3601,8 +3601,10 @@ def is_known_model(model: Optional[str], llm_router: Optional[Router]) -> bool: return False model_names = llm_router.get_model_names() + model_names_set = set(model_names) + is_in_list = False - if model in model_names: + if model in model_names_set: is_in_list = True return is_in_list