mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
feat(llm_passthrough_endpoints.py): add router model support on azure passthrough
This commit is contained in:
Krrish Dholakia
parent
27c64c90dc
commit
6f2b753a5d
4 changed files with 461 additions and 40 deletions
|
|
@ -22173,6 +22173,307 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/bigscience/mt0-xxl-13b": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.0005,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/core42/jais-13b-chat": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.0005,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/google/flan-t5-xl-3b": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.0001,
|
||||
"output_cost_per_token": 0.00025,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-13b-chat-v2": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.0005,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-13b-instruct-v2": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.0005,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-3-3-8b-instruct": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00025,
|
||||
"output_cost_per_token": 0.001,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-4-h-small": {
|
||||
"max_tokens": 20480,
|
||||
"max_input_tokens": 20480,
|
||||
"max_output_tokens": 20480,
|
||||
"input_cost_per_token": 0.000625,
|
||||
"output_cost_per_token": 0.0025,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-guardian-3-2-2b": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00015,
|
||||
"output_cost_per_token": 0.0006,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-guardian-3-3-8b": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00025,
|
||||
"output_cost_per_token": 0.001,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-ttm-1024-96-r2": {
|
||||
"max_tokens": 512,
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"input_cost_per_token": 0.000625,
|
||||
"output_cost_per_token": 0.000625,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-ttm-1536-96-r2": {
|
||||
"max_tokens": 512,
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"input_cost_per_token": 0.000625,
|
||||
"output_cost_per_token": 0.000625,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-ttm-512-96-r2": {
|
||||
"max_tokens": 512,
|
||||
"max_input_tokens": 512,
|
||||
"max_output_tokens": 512,
|
||||
"input_cost_per_token": 0.000625,
|
||||
"output_cost_per_token": 0.000625,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/ibm/granite-vision-3-2-2b": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.00015,
|
||||
"output_cost_per_token": 0.0006,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"watsonx/meta-llama/llama-3-2-11b-vision-instruct": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.00025,
|
||||
"output_cost_per_token": 0.001,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"watsonx/meta-llama/llama-3-2-1b-instruct": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.0001,
|
||||
"output_cost_per_token": 0.0002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/meta-llama/llama-3-2-3b-instruct": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.00015,
|
||||
"output_cost_per_token": 0.0006,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/meta-llama/llama-3-2-90b-vision-instruct": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.002,
|
||||
"output_cost_per_token": 0.008,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"watsonx/meta-llama/llama-3-3-70b-instruct": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.002,
|
||||
"output_cost_per_token": 0.006,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/meta-llama/llama-4-maverick-17b": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.0005,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/meta-llama/llama-guard-3-11b-vision": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.00025,
|
||||
"output_cost_per_token": 0.001,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"watsonx/mistralai/mistral-medium-2505": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.00225,
|
||||
"output_cost_per_token": 0.00675,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/mistralai/mistral-small-2503": {
|
||||
"max_tokens": 32000,
|
||||
"max_input_tokens": 32000,
|
||||
"max_output_tokens": 32000,
|
||||
"input_cost_per_token": 0.0002,
|
||||
"output_cost_per_token": 0.0006,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/mistralai/pixtral-12b-2409": {
|
||||
"max_tokens": 128000,
|
||||
"max_input_tokens": 128000,
|
||||
"max_output_tokens": 128000,
|
||||
"input_cost_per_token": 0.00015,
|
||||
"output_cost_per_token": 0.00015,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"watsonx/openai/gpt-oss-120b": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.004,
|
||||
"output_cost_per_token": 0.016,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
"watsonx/sdaia/allam-1-13b-instruct": {
|
||||
"max_tokens": 8192,
|
||||
"max_input_tokens": 8192,
|
||||
"max_output_tokens": 8192,
|
||||
"input_cost_per_token": 0.0005,
|
||||
"output_cost_per_token": 0.002,
|
||||
"litellm_provider": "watsonx",
|
||||
"mode": "chat",
|
||||
"supports_function_calling": false,
|
||||
"supports_parallel_function_calling": false,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
||||
"whisper-1": {
|
||||
"input_cost_per_second": 0.0001,
|
||||
"litellm_provider": "openai",
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ model_list:
|
|||
api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5"
|
||||
api_key: dummy
|
||||
|
||||
|
||||
# mcp_servers:
|
||||
# github_mcp:
|
||||
# url: "https://api.githubcopilot.com/mcp"
|
||||
|
|
|
|||
|
|
@ -21,9 +21,7 @@ from litellm.constants import BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES
|
|||
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
|
||||
from litellm.proxy._types import *
|
||||
from litellm.proxy.auth.route_checks import RouteChecks
|
||||
from litellm.proxy.auth.user_api_key_auth import (
|
||||
user_api_key_auth,
|
||||
)
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.http_parsing_utils import (
|
||||
_read_request_body,
|
||||
get_form_data,
|
||||
|
|
@ -31,6 +29,7 @@ from litellm.proxy.common_utils.http_parsing_utils import (
|
|||
)
|
||||
from litellm.proxy.pass_through_endpoints.common_utils import get_litellm_virtual_key
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
HttpPassThroughEndpointHelpers,
|
||||
create_pass_through_route,
|
||||
create_websocket_passthrough_route,
|
||||
websocket_passthrough_request,
|
||||
|
|
@ -57,7 +56,9 @@ def create_request_copy(request: Request):
|
|||
}
|
||||
|
||||
|
||||
def is_passthrough_request_using_router_model(request_body: dict, llm_router: Optional[litellm.Router]) -> bool:
|
||||
def is_passthrough_request_using_router_model(
|
||||
request_body: dict, llm_router: Optional[litellm.Router]
|
||||
) -> bool:
|
||||
"""
|
||||
Returns True if the model is in the llm_router model names
|
||||
"""
|
||||
|
|
@ -93,12 +94,16 @@ async def llm_passthrough_factory_proxy_route(
|
|||
model=None,
|
||||
)
|
||||
if provider_config is None:
|
||||
raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Provider {custom_llm_provider} not found"
|
||||
)
|
||||
|
||||
base_target_url = provider_config.get_api_base()
|
||||
|
||||
if base_target_url is None:
|
||||
raise HTTPException(status_code=404, detail=f"Provider {custom_llm_provider} api base not found")
|
||||
raise HTTPException(
|
||||
status_code=404, detail=f"Provider {custom_llm_provider} api base not found"
|
||||
)
|
||||
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
|
|
@ -177,11 +182,17 @@ async def gemini_proxy_route(
|
|||
[Docs](https://docs.litellm.ai/docs/pass_through/google_ai_studio)
|
||||
"""
|
||||
## CHECK FOR LITELLM API KEY IN THE QUERY PARAMS - ?..key=LITELLM_API_KEY
|
||||
google_ai_studio_api_key = request.query_params.get("key") or request.headers.get("x-goog-api-key")
|
||||
google_ai_studio_api_key = request.query_params.get("key") or request.headers.get(
|
||||
"x-goog-api-key"
|
||||
)
|
||||
|
||||
user_api_key_dict = await user_api_key_auth(request=request, api_key=f"Bearer {google_ai_studio_api_key}")
|
||||
user_api_key_dict = await user_api_key_auth(
|
||||
request=request, api_key=f"Bearer {google_ai_studio_api_key}"
|
||||
)
|
||||
|
||||
base_target_url = os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com"
|
||||
base_target_url = (
|
||||
os.getenv("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com"
|
||||
)
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
|
|
@ -293,13 +304,12 @@ async def vllm_proxy_route(
|
|||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/vllm)
|
||||
"""
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
|
||||
HttpPassThroughEndpointHelpers,
|
||||
)
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
request_body = await get_request_body(request)
|
||||
is_router_model = is_passthrough_request_using_router_model(request_body, llm_router)
|
||||
is_router_model = is_passthrough_request_using_router_model(
|
||||
request_body, llm_router
|
||||
)
|
||||
is_streaming_request = is_passthrough_request_streaming(request_body)
|
||||
if is_router_model and llm_router:
|
||||
result = cast(
|
||||
|
|
@ -314,7 +324,11 @@ async def vllm_proxy_route(
|
|||
content=None,
|
||||
data=None,
|
||||
files=None,
|
||||
json=(request_body if request.headers.get("content-type") == "application/json" else None),
|
||||
json=(
|
||||
request_body
|
||||
if request.headers.get("content-type") == "application/json"
|
||||
else None
|
||||
),
|
||||
params=None,
|
||||
headers=None,
|
||||
cookies=None,
|
||||
|
|
@ -492,7 +506,9 @@ async def handle_bedrock_count_tokens(
|
|||
# Extract model from request body
|
||||
model = request_body.get("model")
|
||||
if not model:
|
||||
raise HTTPException(status_code=400, detail={"error": "Model is required in request body"})
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"error": "Model is required in request body"}
|
||||
)
|
||||
|
||||
# Get model parameters from router
|
||||
litellm_params = {"user_api_key_dict": user_api_key_dict}
|
||||
|
|
@ -531,7 +547,9 @@ async def handle_bedrock_count_tokens(
|
|||
raise
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error(f"Error in handle_bedrock_count_tokens: {str(e)}")
|
||||
raise HTTPException(status_code=500, detail={"error": f"CountTokens processing error: {str(e)}"})
|
||||
raise HTTPException(
|
||||
status_code=500, detail={"error": f"CountTokens processing error: {str(e)}"}
|
||||
)
|
||||
|
||||
|
||||
async def bedrock_llm_proxy_route(
|
||||
|
|
@ -583,7 +601,8 @@ async def bedrock_llm_proxy_route(
|
|||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": "Model missing from endpoint. Expected format: /model/<Model>/<endpoint>. Got: " + endpoint,
|
||||
"error": "Model missing from endpoint. Expected format: /model/<Model>/<endpoint>. Got: "
|
||||
+ endpoint,
|
||||
},
|
||||
)
|
||||
|
||||
|
|
@ -647,7 +666,9 @@ async def bedrock_proxy_route(
|
|||
|
||||
aws_region_name = litellm.utils.get_secret(secret_name="AWS_REGION_NAME")
|
||||
if _is_bedrock_agent_runtime_route(endpoint=endpoint): # handle bedrock agents
|
||||
base_target_url = f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
|
||||
base_target_url = (
|
||||
f"https://bedrock-agent-runtime.{aws_region_name}.amazonaws.com"
|
||||
)
|
||||
else:
|
||||
return await bedrock_llm_proxy_route(
|
||||
endpoint=endpoint,
|
||||
|
|
@ -677,7 +698,9 @@ async def bedrock_proxy_route(
|
|||
data = await request.json()
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail={"error": e})
|
||||
_request = AWSRequest(method="POST", url=str(updated_url), data=json.dumps(data), headers=headers)
|
||||
_request = AWSRequest(
|
||||
method="POST", url=str(updated_url), data=json.dumps(data), headers=headers
|
||||
)
|
||||
sigv4.add_auth(_request)
|
||||
prepped = _request.prepare()
|
||||
|
||||
|
|
@ -738,8 +761,14 @@ async def assemblyai_proxy_route(
|
|||
[Docs](https://api.assemblyai.com)
|
||||
"""
|
||||
# Set base URL based on the route
|
||||
assembly_region = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url(url=str(request.url))
|
||||
base_target_url = AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region(region=assembly_region)
|
||||
assembly_region = AssemblyAIPassthroughLoggingHandler._get_assembly_region_from_url(
|
||||
url=str(request.url)
|
||||
)
|
||||
base_target_url = (
|
||||
AssemblyAIPassthroughLoggingHandler._get_assembly_base_url_from_region(
|
||||
region=assembly_region
|
||||
)
|
||||
)
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
if not encoded_endpoint.startswith("/"):
|
||||
|
|
@ -794,17 +823,79 @@ async def azure_proxy_route(
|
|||
Call any azure endpoint using the proxy.
|
||||
|
||||
Just use `{PROXY_BASE_URL}/azure/{endpoint:path}`
|
||||
|
||||
Checks if the deployment id in the url is a litellm model name. If so, it will route using the llm_router.allm_passthrough_route.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
parts = endpoint.split(
|
||||
"/"
|
||||
) # azure model is in the url - e.g. https://{endpoint}/openai/deployments/{deployment-id}/completions?api-version=2024-10-21
|
||||
|
||||
if len(parts) > 1 and llm_router:
|
||||
for part in parts:
|
||||
is_router_model = is_passthrough_request_using_router_model(
|
||||
request_body={"model": part}, llm_router=llm_router
|
||||
)
|
||||
if is_router_model:
|
||||
request_body = await get_request_body(request)
|
||||
is_streaming_request = is_passthrough_request_streaming(request_body)
|
||||
result = cast(
|
||||
httpx.Response,
|
||||
await llm_router.allm_passthrough_route(
|
||||
model=part,
|
||||
method=request.method,
|
||||
endpoint=endpoint,
|
||||
request_query_params=request.query_params,
|
||||
request_headers=dict(request.headers),
|
||||
stream=request_body.get("stream", False),
|
||||
content=None,
|
||||
data=None,
|
||||
files=None,
|
||||
json=(
|
||||
request_body
|
||||
if request.headers.get("content-type") == "application/json"
|
||||
else None
|
||||
),
|
||||
params=None,
|
||||
headers=None,
|
||||
cookies=None,
|
||||
),
|
||||
)
|
||||
|
||||
if is_streaming_request:
|
||||
return StreamingResponse(
|
||||
content=result.aiter_bytes(),
|
||||
status_code=result.status_code,
|
||||
headers=HttpPassThroughEndpointHelpers.get_response_headers(
|
||||
headers=result.headers,
|
||||
custom_headers=None,
|
||||
),
|
||||
)
|
||||
|
||||
content = await result.aread()
|
||||
return Response(
|
||||
content=content,
|
||||
status_code=result.status_code,
|
||||
headers=HttpPassThroughEndpointHelpers.get_response_headers(
|
||||
headers=result.headers,
|
||||
custom_headers=None,
|
||||
),
|
||||
)
|
||||
base_target_url = get_secret_str(secret_name="AZURE_API_BASE")
|
||||
if base_target_url is None:
|
||||
raise Exception("Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure.")
|
||||
raise Exception(
|
||||
"Required 'AZURE_API_BASE' in environment to make pass-through calls to Azure."
|
||||
)
|
||||
# Add or update query parameters
|
||||
azure_api_key = passthrough_endpoint_router.get_credentials(
|
||||
custom_llm_provider=litellm.LlmProviders.AZURE.value,
|
||||
region_name=None,
|
||||
)
|
||||
if azure_api_key is None:
|
||||
raise Exception("Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure.")
|
||||
raise Exception(
|
||||
"Required 'AZURE_API_KEY' in environment to make pass-through calls to Azure."
|
||||
)
|
||||
|
||||
return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler(
|
||||
endpoint=endpoint,
|
||||
|
|
@ -828,7 +919,9 @@ class BaseVertexAIPassThroughHandler(ABC):
|
|||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str:
|
||||
def update_base_target_url_with_credential_location(
|
||||
base_target_url: str, vertex_location: Optional[str]
|
||||
) -> str:
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -838,7 +931,9 @@ class VertexAIDiscoveryPassThroughHandler(BaseVertexAIPassThroughHandler):
|
|||
return "https://discoveryengine.googleapis.com/"
|
||||
|
||||
@staticmethod
|
||||
def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str:
|
||||
def update_base_target_url_with_credential_location(
|
||||
base_target_url: str, vertex_location: Optional[str]
|
||||
) -> str:
|
||||
return base_target_url
|
||||
|
||||
|
||||
|
|
@ -848,7 +943,9 @@ class VertexAIPassThroughHandler(BaseVertexAIPassThroughHandler):
|
|||
return get_vertex_base_url(vertex_location)
|
||||
|
||||
@staticmethod
|
||||
def update_base_target_url_with_credential_location(base_target_url: str, vertex_location: Optional[str]) -> str:
|
||||
def update_base_target_url_with_credential_location(
|
||||
base_target_url: str, vertex_location: Optional[str]
|
||||
) -> str:
|
||||
return get_vertex_base_url(vertex_location)
|
||||
|
||||
|
||||
|
|
@ -914,14 +1011,18 @@ async def _base_vertex_proxy_route(
|
|||
location=vertex_location,
|
||||
)
|
||||
|
||||
base_target_url = get_vertex_pass_through_handler.get_default_base_target_url(vertex_location)
|
||||
base_target_url = get_vertex_pass_through_handler.get_default_base_target_url(
|
||||
vertex_location
|
||||
)
|
||||
|
||||
headers_passed_through = False
|
||||
# Use headers from the incoming request if no vertex credentials are found
|
||||
if vertex_credentials is None or vertex_credentials.vertex_project is None:
|
||||
headers = dict(request.headers) or {}
|
||||
headers_passed_through = True
|
||||
verbose_proxy_logger.debug("default_vertex_config not set, incoming request headers %s", headers)
|
||||
verbose_proxy_logger.debug(
|
||||
"default_vertex_config not set, incoming request headers %s", headers
|
||||
)
|
||||
headers.pop("content-length", None)
|
||||
headers.pop("host", None)
|
||||
else:
|
||||
|
|
@ -1087,7 +1188,9 @@ async def openai_proxy_route(
|
|||
region_name=None,
|
||||
)
|
||||
if openai_api_key is None:
|
||||
raise Exception("Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI.")
|
||||
raise Exception(
|
||||
"Required 'OPENAI_API_KEY' in environment to make pass-through calls to OpenAI."
|
||||
)
|
||||
|
||||
return await BaseOpenAIPassThroughHandler._base_openai_pass_through_handler(
|
||||
endpoint=endpoint,
|
||||
|
|
@ -1133,7 +1236,9 @@ class BaseOpenAIPassThroughHandler:
|
|||
endpoint_func = create_pass_through_route(
|
||||
endpoint=endpoint,
|
||||
target=str(updated_url),
|
||||
custom_headers=BaseOpenAIPassThroughHandler._assemble_headers(api_key=api_key, request=request),
|
||||
custom_headers=BaseOpenAIPassThroughHandler._assemble_headers(
|
||||
api_key=api_key, request=request
|
||||
),
|
||||
) # dynamically construct pass-through endpoint based on incoming path
|
||||
received_value = await endpoint_func(
|
||||
request,
|
||||
|
|
@ -1150,7 +1255,10 @@ class BaseOpenAIPassThroughHandler:
|
|||
"""
|
||||
Appends the OpenAI-Beta header to the headers if the request is an OpenAI Assistants API request
|
||||
"""
|
||||
if RouteChecks._is_assistants_api_request(request) is True and "OpenAI-Beta" not in headers:
|
||||
if (
|
||||
RouteChecks._is_assistants_api_request(request) is True
|
||||
and "OpenAI-Beta" not in headers
|
||||
):
|
||||
headers["OpenAI-Beta"] = "assistants=v2"
|
||||
return headers
|
||||
|
||||
|
|
@ -1166,7 +1274,9 @@ class BaseOpenAIPassThroughHandler:
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _join_url_paths(base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders) -> str:
|
||||
def _join_url_paths(
|
||||
base_url: httpx.URL, path: str, custom_llm_provider: litellm.LlmProviders
|
||||
) -> str:
|
||||
"""
|
||||
Properly joins a base URL with a path, preserving any existing path in the base URL.
|
||||
"""
|
||||
|
|
@ -1182,9 +1292,14 @@ class BaseOpenAIPassThroughHandler:
|
|||
joined_path_str = str(base_url.copy_with(path=full_path))
|
||||
|
||||
# Apply OpenAI-specific path handling for both branches
|
||||
if custom_llm_provider == litellm.LlmProviders.OPENAI and "/v1/" not in joined_path_str:
|
||||
if (
|
||||
custom_llm_provider == litellm.LlmProviders.OPENAI
|
||||
and "/v1/" not in joined_path_str
|
||||
):
|
||||
# Insert v1 after api.openai.com for OpenAI requests
|
||||
joined_path_str = joined_path_str.replace("api.openai.com/", "api.openai.com/v1/")
|
||||
joined_path_str = joined_path_str.replace(
|
||||
"api.openai.com/", "api.openai.com/v1/"
|
||||
)
|
||||
|
||||
return joined_path_str
|
||||
|
||||
|
|
@ -1231,9 +1346,7 @@ async def vertex_ai_live_websocket_passthrough(
|
|||
|
||||
if vertex_credentials_config is not None:
|
||||
resolved_project = resolved_project or vertex_credentials_config.vertex_project
|
||||
temp_location = (
|
||||
resolved_location or vertex_credentials_config.vertex_location
|
||||
)
|
||||
temp_location = resolved_location or vertex_credentials_config.vertex_location
|
||||
# Ensure resolved_location is a string
|
||||
if isinstance(temp_location, dict):
|
||||
resolved_location = str(temp_location)
|
||||
|
|
@ -1241,7 +1354,11 @@ async def vertex_ai_live_websocket_passthrough(
|
|||
resolved_location = str(temp_location)
|
||||
else:
|
||||
resolved_location = None
|
||||
credentials_value = str(vertex_credentials_config.vertex_credentials) if vertex_credentials_config.vertex_credentials is not None else None
|
||||
credentials_value = (
|
||||
str(vertex_credentials_config.vertex_credentials)
|
||||
if vertex_credentials_config.vertex_credentials is not None
|
||||
else None
|
||||
)
|
||||
|
||||
try:
|
||||
resolved_location = resolved_location or (
|
||||
|
|
@ -1302,7 +1419,7 @@ async def vertex_ai_live_websocket_passthrough(
|
|||
# Use the new WebSocket passthrough pattern
|
||||
if user_api_key_dict is None:
|
||||
raise ValueError("user_api_key_dict is required for WebSocket passthrough")
|
||||
|
||||
|
||||
return await websocket_passthrough_request(
|
||||
websocket=websocket,
|
||||
target=service_url,
|
||||
|
|
|
|||
|
|
@ -3601,8 +3601,10 @@ def is_known_model(model: Optional[str], llm_router: Optional[Router]) -> bool:
|
|||
return False
|
||||
model_names = llm_router.get_model_names()
|
||||
|
||||
model_names_set = set(model_names)
|
||||
|
||||
is_in_list = False
|
||||
if model in model_names:
|
||||
if model in model_names_set:
|
||||
is_in_list = True
|
||||
|
||||
return is_in_list
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue