diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index bc3e0680f87..e4e180727d2 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,11 +3,11 @@ model_list: litellm_params: model: "*" -general_settings: - master_key: sk-1234 - pass_through_endpoints: - - path: "/api/public/ingestion" # route you want to add to LiteLLM Proxy Server - target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward - headers: - LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_PUBLIC_KEY" # your langfuse account public key - LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_SECRET_KEY" # your langfuse account secret key \ No newline at end of file +# general_settings: +# master_key: sk-1234 +# pass_through_endpoints: +# - path: "/api/public/ingestion" # route you want to add to LiteLLM Proxy Server +# target: "https://us.cloud.langfuse.com/api/public/ingestion" # URL this route should forward +# headers: +# LANGFUSE_PUBLIC_KEY: "os.environ/LANGFUSE_PUBLIC_KEY" # your langfuse account public key +# LANGFUSE_SECRET_KEY: "os.environ/LANGFUSE_SECRET_KEY" # your langfuse account secret key \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index deb496f2e91..cb60fa5f048 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -5,10 +5,10 @@ import sys import uuid from dataclasses import fields from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, TypedDict, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Extra, Field, Json, model_validator -from typing_extensions import Annotated +from typing_extensions import Annotated, TypedDict from litellm.types.router import UpdateRouterConfig from litellm.types.utils import ProviderField @@ -1082,6 +1082,12 @@ class DynamoDBArgs(LiteLLMBase): assume_role_aws_session_name: Optional[str] = None +class PassThroughEndpointTypedDict(TypedDict): + path: str + target: str + headers: dict + + class ConfigFieldUpdate(LiteLLMBase): field_name: str field_value: Any @@ -1093,6 +1099,13 @@ class ConfigFieldDelete(LiteLLMBase): field_name: str +class FieldDetail(BaseModel): + field_name: str + field_type: str + field_description: str + field_default_value: Any = None + + class ConfigList(LiteLLMBase): field_name: str field_type: str @@ -1101,6 +1114,9 @@ class ConfigList(LiteLLMBase): stored_in_db: Optional[bool] field_default_value: Any premium_field: bool = False + nested_fields: Optional[List[FieldDetail]] = ( + None # For nested dictionary or Pydantic fields + ) class ConfigGeneralSettings(LiteLLMBase): @@ -1203,6 +1219,10 @@ class ConfigGeneralSettings(LiteLLMBase): default=False, description="Public model hub for users to see what models they have access to, supported openai params, etc.", ) + pass_through_endpoints: Optional[PassThroughEndpointTypedDict] = Field( + default=None, + description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through", + ) class ConfigYAML(LiteLLMBase): diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c79a18a5ccd..2213e348d1a 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -13,7 +13,15 @@ import traceback import uuid import warnings from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Any, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + List, + Optional, + get_args, + get_origin, + get_type_hints, +) import requests @@ -548,6 +556,20 @@ async def check_request_disconnection(request: Request, llm_api_call_task): ) +def _resolve_typed_dict_type(typ): + """Resolve the actual TypedDict class from a potentially wrapped type.""" + from typing_extensions import _TypedDictMeta # type: ignore + + origin = get_origin(typ) + if origin is Union: # Check if it's a Union (like Optional) + for arg in get_args(typ): + if isinstance(arg, _TypedDictMeta): + return arg + elif isinstance(typ, type) and isinstance(typ, dict): + return typ + return None + + def prisma_setup(database_url: Optional[str]): global prisma_client, proxy_logging_obj, user_api_key_cache @@ -9409,6 +9431,7 @@ async def get_config_list( "global_max_parallel_requests": {"type": "Integer"}, "max_request_size_mb": {"type": "Integer"}, "max_response_size_mb": {"type": "Integer"}, + "pass_through_endpoints": {"type": "TypedDictionary"}, } return_val = [] @@ -9416,6 +9439,33 @@ async def get_config_list( for field_name, field_info in ConfigGeneralSettings.model_fields.items(): if field_name in allowed_args: + ## HANDLE TYPED DICT + + typed_dict_type = allowed_args[field_name]["type"] + + if typed_dict_type == "TypedDictionary": + typed_dict_class: Optional[Any] = _resolve_typed_dict_type( + field_info.annotation + ) + + if typed_dict_class is None: + nested_fields = None + else: + # Get type hints from the TypedDict to create FieldDetail objects + nested_fields = [ + FieldDetail( + field_name=sub_field, + field_type=type_hint.__name__, + field_description="", # Add custom logic if descriptions are available + field_default_value=general_settings.get(sub_field, None), + ) + for sub_field, type_hint in get_type_hints( + typed_dict_class + ).items() + ] + else: + nested_fields = None + _stored_in_db = None if field_name in db_general_settings_dict: _stored_in_db = True @@ -9429,6 +9479,7 @@ async def get_config_list( field_value=general_settings.get(field_name, None), stored_in_db=_stored_in_db, field_default_value=field_info.default, + nested_fields=nested_fields, ) return_val.append(_response_obj)